From 287da369fcd783a96743ec64d0e4ee515e9fe812 Mon Sep 17 00:00:00 2001 From: baldurk Date: Mon, 22 Mar 2021 11:16:14 +0000 Subject: [PATCH] Fix handling of child processes on linux * We need to ensure we remodify LD_LIBRARY_PATH and LD_PRELOAD before fork/exec if the application (say bash running a script) has overwritten them. We also don't want these to be accidentally inherited into children if we're not hooking children - the same for the vulkan env var, which can't be unset immediately on process injection like the others because it needs to hang around indefinitely. --- renderdoc/common/globalconfig.h | 1 + renderdoc/driver/vulkan/vk_layer.cpp | 8 +- renderdoc/driver/vulkan/vk_replay.cpp | 2 +- renderdoc/os/posix/linux/linux_hook.cpp | 129 +++++++++++- renderdoc/os/posix/posix_libentry.cpp | 4 +- renderdoc/os/posix/posix_process.cpp | 248 +++++++++++++++++------- renderdoc/renderdoc.version | 3 + 7 files changed, 322 insertions(+), 73 deletions(-) diff --git a/renderdoc/common/globalconfig.h b/renderdoc/common/globalconfig.h index 3b773ae42..4ade4b2d4 100644 --- a/renderdoc/common/globalconfig.h +++ b/renderdoc/common/globalconfig.h @@ -165,6 +165,7 @@ enum }; #define RENDERDOC_VULKAN_LAYER_NAME "VK_LAYER_RENDERDOC_Capture" +#define RENDERDOC_VULKAN_LAYER_VAR "ENABLE_VULKAN_RENDERDOC_CAPTURE" #define RENDERDOC_ANDROID_LIBRARY "libVkLayer_GLES_RenderDoc.so" diff --git a/renderdoc/driver/vulkan/vk_layer.cpp b/renderdoc/driver/vulkan/vk_layer.cpp index d67980420..e92697133 100644 --- a/renderdoc/driver/vulkan/vk_layer.cpp +++ b/renderdoc/driver/vulkan/vk_layer.cpp @@ -101,8 +101,8 @@ class VulkanHook : LibraryHook // we don't register any library or function hooks because we use the layer system // we assume the implicit layer is registered - the UI will prompt the user about installing it. - Process::RegisterEnvironmentModification(EnvironmentModification( - EnvMod::Set, EnvSep::NoSep, "ENABLE_VULKAN_RENDERDOC_CAPTURE", "1")); + Process::RegisterEnvironmentModification( + EnvironmentModification(EnvMod::Set, EnvSep::NoSep, RENDERDOC_VULKAN_LAYER_VAR, "1")); // RTSS layer is buggy, disable it to avoid bug reports that are caused by it Process::RegisterEnvironmentModification( @@ -139,8 +139,8 @@ class VulkanHook : LibraryHook void RemoveHooks() { // unset the vulkan layer environment variable - Process::RegisterEnvironmentModification(EnvironmentModification( - EnvMod::Set, EnvSep::NoSep, "ENABLE_VULKAN_RENDERDOC_CAPTURE", "0")); + Process::RegisterEnvironmentModification( + EnvironmentModification(EnvMod::Set, EnvSep::NoSep, RENDERDOC_VULKAN_LAYER_VAR, "0")); Process::ApplyEnvironmentModification(); } diff --git a/renderdoc/driver/vulkan/vk_replay.cpp b/renderdoc/driver/vulkan/vk_replay.cpp index 3614b49c3..69fb9b352 100644 --- a/renderdoc/driver/vulkan/vk_replay.cpp +++ b/renderdoc/driver/vulkan/vk_replay.cpp @@ -4133,7 +4133,7 @@ ReplayStatus Vulkan_CreateReplayDevice(RDCFile *rdc, const ReplayOptions &opts, // disable the layer env var, just in case the user left it set from a previous capture run Process::RegisterEnvironmentModification( - EnvironmentModification(EnvMod::Set, EnvSep::NoSep, "ENABLE_VULKAN_RENDERDOC_CAPTURE", "0")); + EnvironmentModification(EnvMod::Set, EnvSep::NoSep, RENDERDOC_VULKAN_LAYER_VAR, "0")); // disable buggy and user-hostile NV optimus layer, which can completely delete physical devices // (not just rearrange them) and cause problems between capture and replay. diff --git a/renderdoc/os/posix/linux/linux_hook.cpp b/renderdoc/os/posix/linux/linux_hook.cpp index e9f9361ea..8bd3d1309 100644 --- a/renderdoc/os/posix/linux/linux_hook.cpp +++ b/renderdoc/os/posix/linux/linux_hook.cpp @@ -45,9 +45,15 @@ void *intercept_dlopen(const char *filename, int flag, void *ret); void plthook_lib(void *handle); typedef pid_t (*FORKPROC)(); +typedef int (*EXECLEPROC)(const char *pathname, const char *arg, ...); +typedef int (*EXECVEPROC)(const char *pathname, char *const argv[], char *const envp[]); +typedef int (*EXECVPEPROC)(const char *pathname, char *const argv[], char *const envp[]); typedef void *(*DLOPENPROC)(const char *, int); typedef void *(*DLSYMPROC)(void *, const char *); DLOPENPROC realdlopen = NULL; +EXECLEPROC realexecle = NULL; +EXECVEPROC realexecve = NULL; +EXECVPEPROC realexecvpe = NULL; FORKPROC realfork = NULL; DLSYMPROC realdlsym = NULL; @@ -83,21 +89,136 @@ __attribute__((visibility("default"))) void *dlopen(const char *filename, int fl int GetIdentPort(pid_t childPid); +void PreForkConfigureHooks(); +void GetUnhookedEnvp(char *const *envp, rdcstr &envpStr, rdcarray &modifiedEnv); +void GetHookedEnvp(char *const *envp, rdcstr &envpStr, rdcarray &modifiedEnv); +void ResetHookingEnvVars(); void StopAtMainInChild(); bool StopChildAtMain(pid_t childPid); void ResumeProcess(pid_t childPid, uint32_t delay = 0); +__attribute__((visibility("default"))) int execle(const char *pathname, const char *arg, ...) +{ + va_list args; + va_start(args, arg); + + rdcarray argList; + argList.push_back((char *)arg); + + while(true) + { + char *nextArg = va_arg(args, char *); + argList.push_back(nextArg); + + // list is terminated with a NULL + if(!nextArg) + break; + } + + char **envp = va_arg(args, char **); + + va_end(args); + + if(!realexecve) + { + EXECVEPROC passthru = (EXECVEPROC)dlsym(RTLD_NEXT, "execve"); + return passthru(pathname, argList.data(), envp); + } + + rdcarray modifiedEnv; + rdcstr envpStr; + + // if we're not hooking children just call to the real one, but ensure we remove any hooking env + // vars that were kept around after initialisation + if(!RenderDoc::Inst().GetCaptureOptions().hookIntoChildren) + { + GetUnhookedEnvp(envp, envpStr, modifiedEnv); + return realexecve(pathname, argList.data(), modifiedEnv.data()); + } + + GetHookedEnvp(envp, envpStr, modifiedEnv); + return realexecve(pathname, argList.data(), modifiedEnv.data()); +} + +__attribute__((visibility("default"))) int execve(const char *pathname, char *const argv[], + char *const envp[]) +{ + if(!realexecve) + { + EXECVEPROC passthru = (EXECVEPROC)dlsym(RTLD_NEXT, "execve"); + return passthru(pathname, argv, envp); + } + + rdcarray modifiedEnv; + rdcstr envpStr; + + // if we're not hooking children just call to the real one, but ensure we remove any hooking env + // vars that were kept around after initialisation + if(!RenderDoc::Inst().GetCaptureOptions().hookIntoChildren) + { + GetUnhookedEnvp(envp, envpStr, modifiedEnv); + return realexecve(pathname, argv, envp); + } + + GetHookedEnvp(envp, envpStr, modifiedEnv); + return realexecve(pathname, argv, modifiedEnv.data()); +} + +__attribute__((visibility("default"))) int execvpe(const char *pathname, char *const argv[], + char *const envp[]) +{ + if(!realexecvpe) + { + EXECVPEPROC passthru = (EXECVPEPROC)dlsym(RTLD_NEXT, "execvpe"); + return passthru(pathname, argv, envp); + } + + rdcarray modifiedEnv; + rdcstr envpStr; + + // if we're not hooking children just call to the real one, but ensure we remove any hooking env + // vars that were kept around after initialisation + if(!RenderDoc::Inst().GetCaptureOptions().hookIntoChildren) + { + GetUnhookedEnvp(envp, envpStr, modifiedEnv); + return realexecvpe(pathname, argv, envp); + } + + GetHookedEnvp(envp, envpStr, modifiedEnv); + return realexecvpe(pathname, argv, modifiedEnv.data()); +} + __attribute__((visibility("default"))) pid_t fork() { if(!realfork) { FORKPROC passthru = (FORKPROC)dlsym(RTLD_NEXT, "fork"); - return passthru(); } + // if we're not hooking children just call to the real one, we don't have to do anything + if(!RenderDoc::Inst().GetCaptureOptions().hookIntoChildren) + { + // this is a nasty hack. We set this env var when we inject into a process, but because we don't + // know when vulkan may be initialised we need to leave it on indefinitely. If we're not + // injecting into children we need to unset this variable so it doesn't get inherited. + // + // Note this does nothing if the application is doing fork + execve or other variant that passes + // envp because it was probably fetched before the fork - see the exec hooks for where we patch + // that part. + + pid_t ret = realfork(); + if(ret == 0) + unsetenv(RENDERDOC_VULKAN_LAYER_VAR); + + return ret; + } + // fork in a captured application. Need to get the child ident and register it + // set up environment variables for hooking now + PreForkConfigureHooks(); + pid_t ret = realfork(); if(ret == 0) @@ -106,6 +227,9 @@ __attribute__((visibility("default"))) pid_t fork() } else if(ret > 0) { + // restore environment variables + ResetHookingEnvVars(); + bool stopped = StopChildAtMain(ret); if(stopped) @@ -309,6 +433,9 @@ void LibraryHooks::BeginHookRegistration() { realdlopen = (DLOPENPROC)dlsym(RTLD_NEXT, "dlopen"); realfork = (FORKPROC)dlsym(RTLD_NEXT, "fork"); + realexecle = (EXECLEPROC)dlsym(RTLD_NEXT, "execle"); + realexecve = (EXECVEPROC)dlsym(RTLD_NEXT, "execve"); + realexecvpe = (EXECVPEPROC)dlsym(RTLD_NEXT, "execvpe"); } bool LibraryHooks::Detect(const char *identifier) diff --git a/renderdoc/os/posix/posix_libentry.cpp b/renderdoc/os/posix/posix_libentry.cpp index 4b738a14e..1cac38b08 100644 --- a/renderdoc/os/posix/posix_libentry.cpp +++ b/renderdoc/os/posix/posix_libentry.cpp @@ -26,7 +26,7 @@ #include "hooks/hooks.h" #include "os/os_specific.h" -void dlopen_hook_init(); +void ResetHookingEnvVars(); // DllMain equivalent void library_loaded() @@ -45,6 +45,8 @@ void library_loaded() { RenderDoc::Inst().Initialise(); + ResetHookingEnvVars(); + rdcstr capturefile = Process::GetEnvVariable("RENDERDOC_CAPFILE"); rdcstr opts = Process::GetEnvVariable("RENDERDOC_CAPOPTS"); diff --git a/renderdoc/os/posix/posix_process.cpp b/renderdoc/os/posix/posix_process.cpp index d913a32d5..179be1957 100644 --- a/renderdoc/os/posix/posix_process.cpp +++ b/renderdoc/os/posix/posix_process.cpp @@ -37,6 +37,7 @@ #include "api/replay/capture_options.h" #include "api/replay/control_types.h" #include "common/threading.h" +#include "core/core.h" #include "os/os_specific.h" #include "strings/string_utils.h" @@ -262,11 +263,11 @@ static rdcarray &GetEnvModifications() return envCallbacks; } -static std::map EnvStringToEnvMap(const char **envstring) +static std::map EnvStringToEnvMap(char *const *envstring) { std::map ret; - const char **e = envstring; + char *const *e = envstring; while(*e) { @@ -341,6 +342,61 @@ void Process::RegisterEnvironmentModification(const EnvironmentModification &mod GetEnvModifications().push_back(modif); } +void ApplySingleEnvMod(EnvironmentModification &m, rdcstr &value) +{ + switch(m.mod) + { + case EnvMod::Set: value = m.value.c_str(); break; + case EnvMod::Append: + { + if(!value.empty()) + { + if(m.sep == EnvSep::Platform || m.sep == EnvSep::Colon) + value += ":"; + else if(m.sep == EnvSep::SemiColon) + value += ";"; + } + value += m.value.c_str(); + break; + } + case EnvMod::Prepend: + { + if(!value.empty()) + { + rdcstr prep = m.value; + if(m.sep == EnvSep::Platform || m.sep == EnvSep::Colon) + prep += ":"; + else if(m.sep == EnvSep::SemiColon) + prep += ";"; + value = prep + value; + } + else + { + value = m.value.c_str(); + } + break; + } + } +} + +void ApplyEnvironmentModifications(rdcarray &modifications) +{ + // turn environment string to a UTF-8 map + char **currentEnvironment = GetCurrentEnvironment(); + std::map currentEnv = EnvStringToEnvMap(currentEnvironment); + + for(size_t i = 0; i < modifications.size(); i++) + { + EnvironmentModification &m = modifications[i]; + + rdcstr value = currentEnv[m.name.c_str()]; + + ApplySingleEnvMod(m, value); + + setenv(m.name.c_str(), value.c_str(), true); + } +} + // on linux we apply environment changes before launching the program, as // there is no support for injecting/loading renderdoc into a running program // in any way, and we also have some environment changes that we *have* to make @@ -350,53 +406,8 @@ void Process::RegisterEnvironmentModification(const EnvironmentModification &mod // in process (e.g. if we notice a setting and want to enable an env var as a result) void Process::ApplyEnvironmentModification() { - // turn environment string to a UTF-8 map - char **currentEnvironment = GetCurrentEnvironment(); - std::map currentEnv = EnvStringToEnvMap((const char **)currentEnvironment); rdcarray &modifications = GetEnvModifications(); - - for(size_t i = 0; i < modifications.size(); i++) - { - EnvironmentModification &m = modifications[i]; - - rdcstr value = currentEnv[m.name.c_str()]; - - switch(m.mod) - { - case EnvMod::Set: value = m.value.c_str(); break; - case EnvMod::Append: - { - if(!value.empty()) - { - if(m.sep == EnvSep::Platform || m.sep == EnvSep::Colon) - value += ":"; - else if(m.sep == EnvSep::SemiColon) - value += ";"; - } - value += m.value.c_str(); - break; - } - case EnvMod::Prepend: - { - if(!value.empty()) - { - rdcstr prep = m.value; - if(m.sep == EnvSep::Platform || m.sep == EnvSep::Colon) - prep += ":"; - else if(m.sep == EnvSep::SemiColon) - prep += ";"; - value = prep + value; - } - else - { - value = m.value.c_str(); - } - break; - } - } - - setenv(m.name.c_str(), value.c_str(), true); - } + ApplyEnvironmentModifications(modifications); // these have been applied to the current process modifications.clear(); @@ -712,25 +723,9 @@ uint32_t Process::LaunchScript(const rdcstr &script, const rdcstr &workingDir, return LaunchProcess("bash", workingDir, args, internal, result); } -rdcpair Process::LaunchAndInjectIntoProcess( - const rdcstr &app, const rdcstr &workingDir, const rdcstr &cmdLine, - const rdcarray &envList, const rdcstr &capturefile, - const CaptureOptions &opts, bool waitForExit) +void GetHookingEnvMods(rdcarray &modifications, const CaptureOptions &opts, + const rdcstr &capturefile) { - if(app.empty()) - { - RDCERR("Invalid empty 'app'"); - return {ReplayStatus::InternalError, 0}; - } - - // turn environment string to a UTF-8 map - char **currentEnvironment = GetCurrentEnvironment(); - std::map env = EnvStringToEnvMap((const char **)currentEnvironment); - rdcarray modifications = GetEnvModifications(); - - for(const EnvironmentModification &e : envList) - modifications.push_back(e); - rdcstr binpath, libpath, ownlibpath; { FileIO::GetExecutableFilename(binpath); @@ -759,6 +754,12 @@ rdcpair Process::LaunchAndInjectIntoProcess( rdcstr optstr = opts.EncodeAsString(); + modifications.push_back(EnvironmentModification(EnvMod::Append, EnvSep::Platform, + "RENDERDOC_ORIGLIBPATH", + Process::GetEnvVariable(LIB_PATH_ENV_VAR))); + modifications.push_back(EnvironmentModification(EnvMod::Append, EnvSep::Platform, + "RENDERDOC_ORIGPRELOAD", + Process::GetEnvVariable(PRELOAD_ENV_VAR))); modifications.push_back( EnvironmentModification(EnvMod::Append, EnvSep::Platform, LIB_PATH_ENV_VAR, binpath)); modifications.push_back( @@ -773,6 +774,121 @@ rdcpair Process::LaunchAndInjectIntoProcess( EnvironmentModification(EnvMod::Set, EnvSep::NoSep, "RENDERDOC_CAPOPTS", optstr)); modifications.push_back(EnvironmentModification(EnvMod::Set, EnvSep::NoSep, "RENDERDOC_DEBUG_LOG_FILE", RDCGETLOGFILE())); +} + +void PreForkConfigureHooks() +{ + rdcarray modifications; + + GetHookingEnvMods(modifications, RenderDoc::Inst().GetCaptureOptions(), + RenderDoc::Inst().GetCaptureFileTemplate()); + + ApplyEnvironmentModifications(modifications); +} + +void GetUnhookedEnvp(char *const *envp, rdcstr &envpStr, rdcarray &modifiedEnv) +{ + std::map envmap = EnvStringToEnvMap(envp); + + // this is a nasty hack. We set this env var when we inject into a child, but because we don't + // know when vulkan may be initialised we need to leave it on indefinitely. If we're not + // injecting into children we need to unset this variable so it doesn't get inherited. + envmap.erase(RENDERDOC_VULKAN_LAYER_VAR); + + envpStr.clear(); + + // flatten the map to a string + for(auto it = envmap.begin(); it != envmap.end(); it++) + { + envpStr += it->first; + envpStr += "="; + envpStr += it->second; + envpStr.push_back('\0'); + } + envpStr.push_back('\0'); + + // create the array desired + char *c = envpStr.data(); + while(*c) + { + modifiedEnv.push_back(c); + c += strlen(c) + 1; + } + modifiedEnv.push_back(NULL); +} + +void GetHookedEnvp(char *const *envp, rdcstr &envpStr, rdcarray &modifiedEnv) +{ + rdcarray modifications; + + GetHookingEnvMods(modifications, RenderDoc::Inst().GetCaptureOptions(), + RenderDoc::Inst().GetCaptureFileTemplate()); + + std::map envmap = EnvStringToEnvMap(envp); + + for(EnvironmentModification &mod : modifications) + { + // update the values for original values we're storing, since they were gotten by querying the + // *current* environment not envp here. + if(mod.name == "RENDERDOC_ORIGLIBPATH") + mod.value = envmap[LIB_PATH_ENV_VAR]; + else if(mod.name == "RENDERDOC_ORIGPRELOAD") + mod.value = envmap[PRELOAD_ENV_VAR]; + + // modify the map in-place + ApplySingleEnvMod(mod, envmap[mod.name.c_str()]); + } + + envpStr.clear(); + + // flatten the map to a string + for(auto it = envmap.begin(); it != envmap.end(); it++) + { + envpStr += it->first; + envpStr += "="; + envpStr += it->second; + envpStr.push_back('\0'); + } + envpStr.push_back('\0'); + + // create the array desired + char *c = envpStr.data(); + while(*c) + { + modifiedEnv.push_back(c); + c += strlen(c) + 1; + } + modifiedEnv.push_back(NULL); +} + +void ResetHookingEnvVars() +{ + setenv(LIB_PATH_ENV_VAR, Process::GetEnvVariable("RENDERDOC_ORIGLIBPATH").c_str(), true); + setenv(PRELOAD_ENV_VAR, Process::GetEnvVariable("RENDERDOC_ORIGPRELOAD").c_str(), true); + unsetenv("RENDERDOC_ORIGLIBPATH"); + unsetenv("RENDERDOC_ORIGPRELOAD"); +} + +rdcpair Process::LaunchAndInjectIntoProcess( + const rdcstr &app, const rdcstr &workingDir, const rdcstr &cmdLine, + const rdcarray &envList, const rdcstr &capturefile, + const CaptureOptions &opts, bool waitForExit) +{ + if(app.empty()) + { + RDCERR("Invalid empty 'app'"); + return {ReplayStatus::InternalError, 0}; + } + + // turn environment string to a UTF-8 map + char **currentEnvironment = GetCurrentEnvironment(); + std::map env = EnvStringToEnvMap(currentEnvironment); + rdcarray modifications = GetEnvModifications(); + + for(const EnvironmentModification &e : envList) + modifications.push_back(e); + + GetHookingEnvMods(modifications, opts, capturefile); for(size_t i = 0; i < modifications.size(); i++) { diff --git a/renderdoc/renderdoc.version b/renderdoc/renderdoc.version index 4bf00bfe4..c75e39eb3 100644 --- a/renderdoc/renderdoc.version +++ b/renderdoc/renderdoc.version @@ -8,6 +8,9 @@ dlopen; dlsym; fork; + execle; + execve; + execvpe; _exit; RENDERDOC_*; VK_LAYER_RENDERDOC_*;