diff --git a/renderdoc/core/core.cpp b/renderdoc/core/core.cpp index 6a9c779c7..13285991d 100644 --- a/renderdoc/core/core.cpp +++ b/renderdoc/core/core.cpp @@ -195,6 +195,8 @@ void RenderDoc::Initialise() if(!IsReplayApp()) { + Process::ApplyEnvironmentModification(); + uint32_t port = RenderDoc_FirstCaptureNetworkPort; Network::Socket *sock = Network::CreateServerSocket("0.0.0.0", port&0xffff, 4); diff --git a/renderdoc/driver/vulkan/vk_linux.cpp b/renderdoc/driver/vulkan/vk_linux.cpp index eb031e455..79aaa801a 100644 --- a/renderdoc/driver/vulkan/vk_linux.cpp +++ b/renderdoc/driver/vulkan/vk_linux.cpp @@ -164,4 +164,4 @@ VkResult WrappedVulkan::vkCreateXlibSurfaceKHR( void *LoadVulkanLibrary() { return Process::LoadModule("libvulkan.so"); -} +} \ No newline at end of file diff --git a/renderdoc/driver/vulkan/vk_replay.cpp b/renderdoc/driver/vulkan/vk_replay.cpp index b6e50701b..dfadf8c09 100644 --- a/renderdoc/driver/vulkan/vk_replay.cpp +++ b/renderdoc/driver/vulkan/vk_replay.cpp @@ -4410,7 +4410,7 @@ void VulkanReplay::SetProxyBufferData(ResourceId bufid, byte *data, size_t dataS VULKANNOTIMP("SetProxyTextureData"); } -// in vk_replay_platform.cpp +// in vk_.cpp void *LoadVulkanLibrary(); ReplayCreateStatus Vulkan_CreateReplayDevice(const char *logfile, IReplayDriver **driver) diff --git a/renderdoc/driver/vulkan/vk_tracelayer.cpp b/renderdoc/driver/vulkan/vk_tracelayer.cpp index f58bc0c81..66a5e5b56 100644 --- a/renderdoc/driver/vulkan/vk_tracelayer.cpp +++ b/renderdoc/driver/vulkan/vk_tracelayer.cpp @@ -35,9 +35,12 @@ #include "common/common.h" #include "common/threading.h" +#include "serialise/string_utils.h" #include "data/version.h" +#include "os/os_specific.h" + // this should be in the vulkan definition header #ifdef WIN32 #undef VK_LAYER_EXPORT @@ -47,6 +50,20 @@ void InitDeviceTable(const VkBaseLayerObject *obj); void InitInstanceTable(const VkBaseLayerObject *obj); +// in vk_.cpp +bool LayerRegistered(); + +struct RegisterCallback +{ + RegisterCallback() + { + // we assume the implicit layer is registered - the UI will prompt the user about installing it. + Process::RegisterEnvironmentModification(Process::EnvironmentModification(Process::eEnvModification_Replace, "ENABLE_VULKAN_RENDERDOC_CAPTURE", "1")); + } +}; + +static RegisterCallback registercb; + // RenderDoc State // RenderDoc Intercepts, these must all be entry points with a dispatchable object diff --git a/renderdoc/driver/vulkan/vk_win32.cpp b/renderdoc/driver/vulkan/vk_win32.cpp index 782404c3e..ca7a11bb2 100644 --- a/renderdoc/driver/vulkan/vk_win32.cpp +++ b/renderdoc/driver/vulkan/vk_win32.cpp @@ -106,4 +106,4 @@ VkBool32 WrappedVulkan::vkGetPhysicalDeviceWin32PresentationSupportKHR( void *LoadVulkanLibrary() { return Process::LoadModule("vulkan-0.dll"); -} +} \ No newline at end of file diff --git a/renderdoc/os/linux/linux_process.cpp b/renderdoc/os/linux/linux_process.cpp index f43a173af..657b66b42 100644 --- a/renderdoc/os/linux/linux_process.cpp +++ b/renderdoc/os/linux/linux_process.cpp @@ -35,7 +35,50 @@ #include "serialise/string_utils.h" -pid_t RunProcess(const char *app, const char *workingDir, const char *cmdLine, char *const *envp) +static vector &GetEnvModifications() +{ + static vector envCallbacks; + return envCallbacks; +} + +static map EnvStringToEnvMap(const char **envstring) +{ + map ret; + + const char **e = envstring; + + while(*e) + { + const char *equals = strchr(*e, '='); + + string name; + string value; + + name.assign(e, equals); + value = equals+1; + + ret[name] = value; + + e++; + } + + return ret; +} + +void Process::RegisterEnvironmentModification(Process::EnvironmentModification modif) +{ + GetEnvModifications().push_back(modif); +} + +// on linux we apply environment changes before launching the program, as +// there is no support for injecting/loading renderdoc into a running program +// in any way, and we also have some environment changes that we *have* to make +// for correct hooking (LD_LIBRARY_PATH/LD_PRELOAD) +void Process::ApplyEnvironmentModification() +{ +} + +static pid_t RunProcess(const char *app, const char *workingDir, const char *cmdLine, char *const *envp) { if(!app) return (pid_t)0; @@ -212,171 +255,89 @@ uint32_t Process::LaunchAndInjectIntoProcess(const char *app, const char *workin RDCERR("Invalid empty 'app'"); return 0; } - - char **envp = NULL; - - int nenv = 0; - for(; environ[nenv]; nenv++); - - const int numEnvAdd = 7; - // LD_LIBRARY_PATH - // LD_PRELOAD - // VK_LAYER_PATH - // VK_DEVICE_LAYERS - // VK_INSTANCE_LAYERS - // RENDERDOC_CAPTUREOPTS - // RENDERDOC_LOGFILE - - // might find these already existant in the environment - bool libpath = false; - bool preload = false; - bool layerdirs = false; - bool devicelayers = false; - bool instancelayers = false; - - nenv += 1+numEnvAdd; // account for terminating NULL we need to replicate, and up to N additional environment varibales - - envp = new char*[nenv]; - - string localpath; - FileIO::GetExecutableFilename(localpath); - localpath = dirname(localpath); - - int i=0; int srci=0; - for(; i < nenv; srci++) + + // turn environment string to a UTF-8 map + map env = EnvStringToEnvMap(environ); + vector &modifications = GetEnvModifications(); + + string libpath; { - if(environ[srci] == NULL) - { - envp[i] = NULL; - break; - } - - size_t len = strlen(environ[srci])+1; - - if(!strncmp(environ[srci], "LD_LIBRARY_PATH=", sizeof("LD_LIBRARY_PATH=")-1)) - { - libpath = true; - envp[i] = new char[len+localpath.length()+1]; - memcpy(envp[i], environ[srci], len); - strcat(envp[i], ":"); - strcat(envp[i], localpath.c_str()); - } - else if(!strncmp(environ[srci], "LD_PRELOAD=", sizeof("LD_PRELOAD=")-1)) - { - preload = true; - envp[i] = new char[len+sizeof("librenderdoc.so")]; - memcpy(envp[i], environ[srci], len); - strcat(envp[i], ":librenderdoc.so"); - } - else if(!strncmp(environ[srci], "VK_LAYER_PATH=", sizeof("VK_LAYER_PATH=")-1)) - { - layerdirs = true; - envp[i] = new char[len+1+localpath.length()+30]; - memcpy(envp[i], environ[srci], len); - strcat(envp[i], ":"); - strcat(envp[i], localpath.c_str()); - } - else if(!strncmp(environ[srci], "VK_DEVICE_LAYERS=", sizeof("VK_DEVICE_LAYERS=")-1)) - { - devicelayers = true; - envp[i] = new char[len+sizeof(":RenderDoc")]; - memcpy(envp[i], environ[srci], len); - strcat(envp[i], ":RenderDoc"); - } - else if(!strncmp(environ[srci], "VK_INSTANCE_LAYERS=", sizeof("VK_INSTANCE_LAYERS=")-1)) - { - instancelayers = true; - envp[i] = new char[len+sizeof(":RenderDoc")]; - memcpy(envp[i], environ[srci], len); - strcat(envp[i], ":RenderDoc"); - } - else if(!strncmp(environ[srci], "RENDERDOC_", sizeof("RENDERDOC_")-1)) - { - // skip this variable entirely - continue; - } - else - { - // basic copy - envp[i] = new char[len]; - memcpy(envp[i], environ[srci], len); - } - - i++; + FileIO::GetExecutableFilename(libpath); + libpath = dirname(libpath); } - if(!libpath) + string optstr; { - string e = StringFormat::Fmt("LD_LIBRARY_PATH=%s", localpath.c_str()); - envp[i] = new char[e.length()+1]; - memcpy(envp[i], e.c_str(), e.length()+1); - i++; - envp[i] = NULL; - } - - if(!preload) - { - string e = StringFormat::Fmt("LD_PRELOAD=%s/librenderdoc.so", localpath.c_str()); - envp[i] = new char[e.length()+1]; - memcpy(envp[i], e.c_str(), e.length()+1); - i++; - envp[i] = NULL; - } - - if(!layerdirs) - { - string e = StringFormat::Fmt("VK_LAYER_PATH=%s", localpath.c_str()); - envp[i] = new char[e.length()+1]; - memcpy(envp[i], e.c_str(), e.length()+1); - i++; - envp[i] = NULL; - } - - if(!devicelayers) - { - string e = StringFormat::Fmt("VK_DEVICE_LAYERS=RenderDoc"); - envp[i] = new char[e.length()+1]; - memcpy(envp[i], e.c_str(), e.length()+1); - i++; - envp[i] = NULL; - } - - if(!instancelayers) - { - string e = StringFormat::Fmt("VK_INSTANCE_LAYERS=RenderDoc"); - envp[i] = new char[e.length()+1]; - memcpy(envp[i], e.c_str(), e.length()+1); - i++; - envp[i] = NULL; - } - - if(opts) - { - string optstr; + optstr.reserve(sizeof(CaptureOptions)*2+1); + byte *b = (byte *)opts; + for(size_t i=0; i < sizeof(CaptureOptions); i++) { - optstr.reserve(sizeof(CaptureOptions)*2+1); - byte *b = (byte *)opts; - for(size_t i=0; i < sizeof(CaptureOptions); i++) - { - optstr.push_back(char( 'a' + ((b[i] >> 4)&0xf) )); - optstr.push_back(char( 'a' + ((b[i] )&0xf) )); - } + optstr.push_back(char( 'a' + ((b[i] >> 4)&0xf) )); + optstr.push_back(char( 'a' + ((b[i] )&0xf) )); } - - string e = StringFormat::Fmt("RENDERDOC_CAPTUREOPTS=%s", optstr.c_str()); - envp[i] = new char[e.length()+1]; - memcpy(envp[i], e.c_str(), e.length()+1); - i++; - envp[i] = NULL; } - if(logfile) + modifications.push_back(EnvironmentModification(eEnvModification_AppendPlatform, "LD_LIBRARY_PATH", libpath.c_str())); + modifications.push_back(EnvironmentModification(eEnvModification_AppendPlatform, "LD_PRELOAD", "librenderdoc.so")); + modifications.push_back(EnvironmentModification(eEnvModification_Replace, "RENDERDOC_LOGFILE", logfile)); + modifications.push_back(EnvironmentModification(eEnvModification_Replace, "RENDERDOC_CAPTUREOPTS", optstr.c_str())); + + for(size_t i=0; i < modifications.size(); i++) { - string e = StringFormat::Fmt("RENDERDOC_LOGFILE=%s", logfile); - envp[i] = new char[e.length()+1]; - memcpy(envp[i], e.c_str(), e.length()+1); + EnvironmentModification &m = modifications[i]; + + string value = env[m.name]; + + switch(m.type) + { + case eEnvModification_Replace: + value = m.value; + break; + case eEnvModification_Append: + value += m.value; + break; + case eEnvModification_AppendColon: + if(!value.empty()) + value += ":"; + value += m.value; + break; + case eEnvModification_AppendPlatform: + case eEnvModification_AppendSemiColon: + if(!value.empty()) + value += ";"; + value += m.value; + break; + case eEnvModification_Prepend: + value = m.value + value; + break; + case eEnvModification_PrependColon: + if(!value.empty()) + value = m.value + ":" + value; + else + value = m.value; + break; + case eEnvModification_PrependPlatform: + case eEnvModification_PrependSemiColon: + if(!value.empty()) + value = m.value + ";" + value; + else + value = m.value; + break; + default: + RDCERR("Unexpected environment modification type"); + } + } + + char **envp = new char *[env.size()+1]; + envp[env.size()] = NULL; + + int i=0; + for(auto it=env.begin(); it != env.end(); it++) + { + string envline = it->first + "=" + it->second; + envp[i] = new char[envline.size()+1]; + memcpy(envp[i], envline.c_str(), envline.size()+1); i++; - envp[i] = NULL; } pid_t childPid = RunProcess(app, workingDir, cmdLine, envp); diff --git a/renderdoc/os/os_specific.h b/renderdoc/os/os_specific.h index a014ae76c..c065ed30d 100644 --- a/renderdoc/os/os_specific.h +++ b/renderdoc/os/os_specific.h @@ -38,13 +38,42 @@ #include #include +#include using std::string; using std::vector; +using std::map; struct CaptureOptions; namespace Process { + enum ModificationType + { + eEnvModification_Replace = 0, + + // prepend/append options will replace if there is no existing variable + eEnvModification_Append, // append with no separators + eEnvModification_AppendColon, // append, separated by colons + eEnvModification_AppendSemiColon, // append, separated by semi-colons + eEnvModification_AppendPlatform, // append, separated by colons for linux & semi-colons for windows + + eEnvModification_Prepend, // prepend with no separators + eEnvModification_PrependColon, // prepend, separated by colons + eEnvModification_PrependSemiColon, // prepend, separated by semi-colons + eEnvModification_PrependPlatform, // prepend, separated by colons for linux & semi-colons for windows + }; + struct EnvironmentModification + { + EnvironmentModification() : type(eEnvModification_Replace), name(""), value("") {} + EnvironmentModification(ModificationType t, const char *n, const char *v) : type(t), name(n), value(v) {} + ModificationType type; + string name; + string value; + }; + void RegisterEnvironmentModification(EnvironmentModification modif); + + void ApplyEnvironmentModification(); + void StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions *opts); uint32_t InjectIntoProcess(uint32_t pid, const char *logfile, const CaptureOptions *opts, bool waitForExit); uint32_t LaunchProcess(const char *app, const char *workingDir, const char *cmdLine); diff --git a/renderdoc/os/win32/win32_process.cpp b/renderdoc/os/win32/win32_process.cpp index 87045f60d..bc52eb9e0 100644 --- a/renderdoc/os/win32/win32_process.cpp +++ b/renderdoc/os/win32/win32_process.cpp @@ -36,6 +36,104 @@ #include using std::string; +static vector &GetEnvModifications() +{ + static vector envCallbacks; + return envCallbacks; +} + +static map EnvStringToEnvMap(const wchar_t *envstring) +{ + map ret; + + const wchar_t *e = envstring; + + while(*e) + { + const wchar_t *equals = wcschr(e, L'='); + + wstring name; + wstring value; + + name.assign(e, equals); + value = equals+1; + + ret[StringFormat::Wide2UTF8(name)] = StringFormat::Wide2UTF8(value); + + e += name.size(); // jump to = + e++; // advance past it + e += value.size(); // jump to \0 + e++; // advance past it + } + + return ret; +} + +void Process::RegisterEnvironmentModification(Process::EnvironmentModification modif) +{ + GetEnvModifications().push_back(modif); +} + +// on windows we apply environment changes here, after process initialisation +// but before any real work (in RenderDoc::Initialise) so that we support +// injecting the dll into processes we didn't launch (ie didn't control the +// starting environment for), or even the application loading the dll itself +// without any interaction with our replay app. +void Process::ApplyEnvironmentModification() +{ + // turn environment string to a UTF-8 map + map currentEnv = EnvStringToEnvMap(GetEnvironmentStringsW()); + vector &modifications = GetEnvModifications(); + + for(size_t i=0; i < modifications.size(); i++) + { + EnvironmentModification &m = modifications[i]; + + string value = currentEnv[m.name]; + + switch(m.type) + { + case eEnvModification_Replace: + value = m.value; + break; + case eEnvModification_Append: + value += m.value; + break; + case eEnvModification_AppendColon: + if(!value.empty()) + value += ":"; + value += m.value; + break; + case eEnvModification_AppendPlatform: + case eEnvModification_AppendSemiColon: + if(!value.empty()) + value += ";"; + value += m.value; + break; + case eEnvModification_Prepend: + value = m.value + value; + break; + case eEnvModification_PrependColon: + if(!value.empty()) + value = m.value + ":" + value; + else + value = m.value; + break; + case eEnvModification_PrependPlatform: + case eEnvModification_PrependSemiColon: + if(!value.empty()) + value = m.value + ";" + value; + else + value = m.value; + break; + default: + RDCERR("Unexpected environment modification type"); + } + + SetEnvironmentVariableW(StringFormat::UTF82Wide(m.name).c_str(), StringFormat::UTF82Wide(value).c_str()); + } +} + // helpers for various shims and dlls etc, not part of the public API extern "C" __declspec(dllexport) void __cdecl RENDERDOC_GetRemoteAccessIdent(uint32_t *ident) @@ -54,7 +152,6 @@ void __cdecl RENDERDOC_SetLogFile(const char *log) { if(log) RenderDoc::Inst().SetLogFile(log); } - void InjectDLL(HANDLE hProcess, wstring libName) { wchar_t dllPath[MAX_PATH + 1] = {0}; @@ -202,116 +299,58 @@ void InjectFunctionCall(HANDLE hProcess, uintptr_t renderdoc_remote, const char static PROCESS_INFORMATION RunProcess(const char *app, const char *workingDir, const char *cmdLine) { - PROCESS_INFORMATION pi; - STARTUPINFO si; - SECURITY_ATTRIBUTES pSec; - SECURITY_ATTRIBUTES tSec; + PROCESS_INFORMATION pi; + STARTUPINFO si; + SECURITY_ATTRIBUTES pSec; + SECURITY_ATTRIBUTES tSec; - RDCEraseEl(pi); - RDCEraseEl(si); - RDCEraseEl(pSec); - RDCEraseEl(tSec); + RDCEraseEl(pi); + RDCEraseEl(si); + RDCEraseEl(pSec); + RDCEraseEl(tSec); - pSec.nLength = sizeof(pSec); - tSec.nLength = sizeof(tSec); + pSec.nLength = sizeof(pSec); + tSec.nLength = sizeof(tSec); - wstring workdir = L""; + wstring workdir = L""; - if (workingDir != NULL && workingDir[0] != 0) - workdir = StringFormat::UTF82Wide(string(workingDir)); - else - workdir = StringFormat::UTF82Wide(dirname(string(app))); + if(workingDir != NULL && workingDir[0] != 0) + workdir = StringFormat::UTF82Wide(string(workingDir)); + else + workdir = StringFormat::UTF82Wide(dirname(string(app))); - wchar_t *paramsAlloc = NULL; + wchar_t *paramsAlloc = NULL; - wstring wapp = StringFormat::UTF82Wide(string(app)); + wstring wapp = StringFormat::UTF82Wide(string(app)); - // CreateProcessW can modify the params, need space. - size_t len = wapp.length() + 10; + // CreateProcessW can modify the params, need space. + size_t len = wapp.length()+10; - wstring wcmd = L""; + wstring wcmd = L""; - if (cmdLine != NULL && cmdLine[0] != 0) - { - wcmd = StringFormat::UTF82Wide(string(cmdLine)); - len += wcmd.length(); - } + if(cmdLine != NULL && cmdLine[0] != 0) + { + wcmd = StringFormat::UTF82Wide(string(cmdLine)); + len += wcmd.length(); + } - paramsAlloc = new wchar_t[len]; + paramsAlloc = new wchar_t[len]; - RDCEraseMem(paramsAlloc, len*sizeof(wchar_t)); + RDCEraseMem(paramsAlloc, len*sizeof(wchar_t)); - wcscpy_s(paramsAlloc, len, L"\""); - wcscat_s(paramsAlloc, len, wapp.c_str()); - wcscat_s(paramsAlloc, len, L"\""); + wcscpy_s(paramsAlloc, len, L"\""); + wcscat_s(paramsAlloc, len, wapp.c_str()); + wcscat_s(paramsAlloc, len, L"\""); - if (cmdLine != NULL && cmdLine[0] != 0) - { - wcscat_s(paramsAlloc, len, L" "); - wcscat_s(paramsAlloc, len, wcmd.c_str()); - } - - // VKTODOLOW refactor all this once loader supports implicit layers - const wchar_t *myEnv = GetEnvironmentStringsW(); - - wstring newEnv; - - // copy up to the terminating "\0\0" - bool sawpath = false; - bool sawdevice = false; - bool sawinstance = false; - while (myEnv[0] != L'\0' || myEnv[1] != L'\0') - { - if (!wcsncmp(&myEnv[0], L"VK_LAYER_PATH=", sizeof("VK_LAYER_PATH=") - 1)) - { - sawpath = true; - newEnv += L"VK_LAYER_PATH="; - newEnv += dirname(StringFormat::UTF82Wide(FileIO::GetReplayAppFilename())); - newEnv += L";"; - myEnv += (sizeof("VK_LAYER_PATH=") - 1); - } - else if (!wcsncmp(&myEnv[0], L"VK_DEVICE_LAYERS=", sizeof("VK_DEVICE_LAYERS=") - 1)) - { - sawdevice = true; - newEnv += L"VK_DEVICE_LAYERS=VK_LAYER_RENDERDOC_Capture;"; - myEnv += (sizeof("VK_DEVICE_LAYERS=") - 1); - } - else if (!wcsncmp(&myEnv[0], L"VK_INSTANCE_LAYERS=", sizeof("VK_INSTANCE_LAYERS=") - 1)) - { - sawinstance = true; - newEnv += L"VK_INSTANCE_LAYERS=VK_LAYER_RENDERDOC_Capture;"; - myEnv += (sizeof("VK_INSTANCE_LAYERS=") - 1); - } - else - newEnv.push_back(*(myEnv++)); - } - - newEnv.push_back(L'\0'); - - if (!sawpath) - { - newEnv += L"VK_LAYER_PATH="; - newEnv += dirname(StringFormat::UTF82Wide(FileIO::GetReplayAppFilename())); - newEnv.push_back(L'\0'); - } - - if (!sawdevice) - { - newEnv += L"VK_DEVICE_LAYERS=VK_LAYER_RENDERDOC_Capture"; - newEnv.push_back(L'\0'); - } - - if (!sawinstance) - { - newEnv += L"VK_INSTANCE_LAYERS=VK_LAYER_RENDERDOC_Capture"; - newEnv.push_back(L'\0'); - } - - newEnv.push_back(L'\0'); + if(cmdLine != NULL && cmdLine[0] != 0) + { + wcscat_s(paramsAlloc, len, L" "); + wcscat_s(paramsAlloc, len, wcmd.c_str()); + } BOOL retValue = CreateProcessW(NULL, paramsAlloc, &pSec, &tSec, false, CREATE_SUSPENDED|CREATE_UNICODE_ENVIRONMENT, - (void *)newEnv.c_str(), workdir.c_str(), &si, &pi); + NULL, workdir.c_str(), &si, &pi); SAFE_DELETE_ARRAY(paramsAlloc); @@ -644,4 +683,3 @@ uint32_t Process::GetCurrentPID() { return (uint32_t)GetCurrentProcessId(); } -