Fix handling of child processes on linux

* We need to ensure we remodify LD_LIBRARY_PATH and LD_PRELOAD before fork/exec
  if the application (say bash running a script) has overwritten them. We also
  don't want these to be accidentally inherited into children if we're not
  hooking children - the same for the vulkan env var, which can't be unset
  immediately on process injection like the others because it needs to hang
  around indefinitely.

renderdoc/common/globalconfig.h

@@ -165,6 +165,7 @@ enum
 };
 
 #define RENDERDOC_VULKAN_LAYER_NAME "VK_LAYER_RENDERDOC_Capture"
+#define RENDERDOC_VULKAN_LAYER_VAR "ENABLE_VULKAN_RENDERDOC_CAPTURE"
 
 #define RENDERDOC_ANDROID_LIBRARY "libVkLayer_GLES_RenderDoc.so"
 

renderdoc/driver/vulkan/vk_layer.cpp

@@ -101,8 +101,8 @@ class VulkanHook : LibraryHook
     // we don't register any library or function hooks because we use the layer system
 
     // we assume the implicit layer is registered - the UI will prompt the user about installing it.
-    Process::RegisterEnvironmentModification(EnvironmentModification(
-        EnvMod::Set, EnvSep::NoSep, "ENABLE_VULKAN_RENDERDOC_CAPTURE", "1"));
+    Process::RegisterEnvironmentModification(
+        EnvironmentModification(EnvMod::Set, EnvSep::NoSep, RENDERDOC_VULKAN_LAYER_VAR, "1"));
 
     // RTSS layer is buggy, disable it to avoid bug reports that are caused by it
     Process::RegisterEnvironmentModification(
@@ -139,8 +139,8 @@ class VulkanHook : LibraryHook
   void RemoveHooks()
   {
     // unset the vulkan layer environment variable
-    Process::RegisterEnvironmentModification(EnvironmentModification(
-        EnvMod::Set, EnvSep::NoSep, "ENABLE_VULKAN_RENDERDOC_CAPTURE", "0"));
+    Process::RegisterEnvironmentModification(
+        EnvironmentModification(EnvMod::Set, EnvSep::NoSep, RENDERDOC_VULKAN_LAYER_VAR, "0"));
     Process::ApplyEnvironmentModification();
   }
 

renderdoc/driver/vulkan/vk_replay.cpp

@@ -4133,7 +4133,7 @@ ReplayStatus Vulkan_CreateReplayDevice(RDCFile *rdc, const ReplayOptions &opts,
 
   // disable the layer env var, just in case the user left it set from a previous capture run
   Process::RegisterEnvironmentModification(
-      EnvironmentModification(EnvMod::Set, EnvSep::NoSep, "ENABLE_VULKAN_RENDERDOC_CAPTURE", "0"));
+      EnvironmentModification(EnvMod::Set, EnvSep::NoSep, RENDERDOC_VULKAN_LAYER_VAR, "0"));
 
   // disable buggy and user-hostile NV optimus layer, which can completely delete physical devices
   // (not just rearrange them) and cause problems between capture and replay.

renderdoc/os/posix/linux/linux_hook.cpp

@@ -45,9 +45,15 @@ void *intercept_dlopen(const char *filename, int flag, void *ret);
 void plthook_lib(void *handle);
 
 typedef pid_t (*FORKPROC)();
+typedef int (*EXECLEPROC)(const char *pathname, const char *arg, ...);
+typedef int (*EXECVEPROC)(const char *pathname, char *const argv[], char *const envp[]);
+typedef int (*EXECVPEPROC)(const char *pathname, char *const argv[], char *const envp[]);
 typedef void *(*DLOPENPROC)(const char *, int);
 typedef void *(*DLSYMPROC)(void *, const char *);
 DLOPENPROC realdlopen = NULL;
+EXECLEPROC realexecle = NULL;
+EXECVEPROC realexecve = NULL;
+EXECVPEPROC realexecvpe = NULL;
 FORKPROC realfork = NULL;
 DLSYMPROC realdlsym = NULL;
 
@@ -83,21 +89,136 @@ __attribute__((visibility("default"))) void *dlopen(const char *filename, int fl
 
 int GetIdentPort(pid_t childPid);
 
+void PreForkConfigureHooks();
+void GetUnhookedEnvp(char *const *envp, rdcstr &envpStr, rdcarray<char *> &modifiedEnv);
+void GetHookedEnvp(char *const *envp, rdcstr &envpStr, rdcarray<char *> &modifiedEnv);
+void ResetHookingEnvVars();
 void StopAtMainInChild();
 bool StopChildAtMain(pid_t childPid);
 void ResumeProcess(pid_t childPid, uint32_t delay = 0);
 
+__attribute__((visibility("default"))) int execle(const char *pathname, const char *arg, ...)
+{
+  va_list args;
+  va_start(args, arg);
+
+  rdcarray<char *> argList;
+  argList.push_back((char *)arg);
+
+  while(true)
+  {
+    char *nextArg = va_arg(args, char *);
+    argList.push_back(nextArg);
+
+    // list is terminated with a NULL
+    if(!nextArg)
+      break;
+  }
+
+  char **envp = va_arg(args, char **);
+
+  va_end(args);
+
+  if(!realexecve)
+  {
+    EXECVEPROC passthru = (EXECVEPROC)dlsym(RTLD_NEXT, "execve");
+    return passthru(pathname, argList.data(), envp);
+  }
+
+  rdcarray<char *> modifiedEnv;
+  rdcstr envpStr;
+
+  // if we're not hooking children just call to the real one, but ensure we remove any hooking env
+  // vars that were kept around after initialisation
+  if(!RenderDoc::Inst().GetCaptureOptions().hookIntoChildren)
+  {
+    GetUnhookedEnvp(envp, envpStr, modifiedEnv);
+    return realexecve(pathname, argList.data(), modifiedEnv.data());
+  }
+
+  GetHookedEnvp(envp, envpStr, modifiedEnv);
+  return realexecve(pathname, argList.data(), modifiedEnv.data());
+}
+
+__attribute__((visibility("default"))) int execve(const char *pathname, char *const argv[],
+                                                  char *const envp[])
+{
+  if(!realexecve)
+  {
+    EXECVEPROC passthru = (EXECVEPROC)dlsym(RTLD_NEXT, "execve");
+    return passthru(pathname, argv, envp);
+  }
+
+  rdcarray<char *> modifiedEnv;
+  rdcstr envpStr;
+
+  // if we're not hooking children just call to the real one, but ensure we remove any hooking env
+  // vars that were kept around after initialisation
+  if(!RenderDoc::Inst().GetCaptureOptions().hookIntoChildren)
+  {
+    GetUnhookedEnvp(envp, envpStr, modifiedEnv);
+    return realexecve(pathname, argv, envp);
+  }
+
+  GetHookedEnvp(envp, envpStr, modifiedEnv);
+  return realexecve(pathname, argv, modifiedEnv.data());
+}
+
+__attribute__((visibility("default"))) int execvpe(const char *pathname, char *const argv[],
+                                                   char *const envp[])
+{
+  if(!realexecvpe)
+  {
+    EXECVPEPROC passthru = (EXECVPEPROC)dlsym(RTLD_NEXT, "execvpe");
+    return passthru(pathname, argv, envp);
+  }
+
+  rdcarray<char *> modifiedEnv;
+  rdcstr envpStr;
+
+  // if we're not hooking children just call to the real one, but ensure we remove any hooking env
+  // vars that were kept around after initialisation
+  if(!RenderDoc::Inst().GetCaptureOptions().hookIntoChildren)
+  {
+    GetUnhookedEnvp(envp, envpStr, modifiedEnv);
+    return realexecvpe(pathname, argv, envp);
+  }
+
+  GetHookedEnvp(envp, envpStr, modifiedEnv);
+  return realexecvpe(pathname, argv, modifiedEnv.data());
+}
+
 __attribute__((visibility("default"))) pid_t fork()
 {
   if(!realfork)
   {
     FORKPROC passthru = (FORKPROC)dlsym(RTLD_NEXT, "fork");
-
     return passthru();
   }
 
+  // if we're not hooking children just call to the real one, we don't have to do anything
+  if(!RenderDoc::Inst().GetCaptureOptions().hookIntoChildren)
+  {
+    // this is a nasty hack. We set this env var when we inject into a process, but because we don't
+    // know when vulkan may be initialised we need to leave it on indefinitely. If we're not
+    // injecting into children we need to unset this variable so it doesn't get inherited.
+    //
+    // Note this does nothing if the application is doing fork + execve or other variant that passes
+    // envp because it was probably fetched before the fork - see the exec hooks for where we patch
+    // that part.
+
+    pid_t ret = realfork();
+    if(ret == 0)
+      unsetenv(RENDERDOC_VULKAN_LAYER_VAR);
+
+    return ret;
+  }
+
   // fork in a captured application. Need to get the child ident and register it
 
+  // set up environment variables for hooking now
+  PreForkConfigureHooks();
+
   pid_t ret = realfork();
 
   if(ret == 0)
@@ -106,6 +227,9 @@ __attribute__((visibility("default"))) pid_t fork()
   }
   else if(ret > 0)
   {
+    // restore environment variables
+    ResetHookingEnvVars();
+
     bool stopped = StopChildAtMain(ret);
 
     if(stopped)
@@ -309,6 +433,9 @@ void LibraryHooks::BeginHookRegistration()
 {
   realdlopen = (DLOPENPROC)dlsym(RTLD_NEXT, "dlopen");
   realfork = (FORKPROC)dlsym(RTLD_NEXT, "fork");
+  realexecle = (EXECLEPROC)dlsym(RTLD_NEXT, "execle");
+  realexecve = (EXECVEPROC)dlsym(RTLD_NEXT, "execve");
+  realexecvpe = (EXECVPEPROC)dlsym(RTLD_NEXT, "execvpe");
 }
 
 bool LibraryHooks::Detect(const char *identifier)

renderdoc/os/posix/posix_libentry.cpp

@@ -26,7 +26,7 @@
 #include "hooks/hooks.h"
 #include "os/os_specific.h"
 
-void dlopen_hook_init();
+void ResetHookingEnvVars();
 
 // DllMain equivalent
 void library_loaded()
@@ -45,6 +45,8 @@ void library_loaded()
   {
     RenderDoc::Inst().Initialise();
 
+    ResetHookingEnvVars();
+
     rdcstr capturefile = Process::GetEnvVariable("RENDERDOC_CAPFILE");
     rdcstr opts = Process::GetEnvVariable("RENDERDOC_CAPOPTS");
 

renderdoc/os/posix/posix_process.cpp

@@ -37,6 +37,7 @@
 #include "api/replay/capture_options.h"
 #include "api/replay/control_types.h"
 #include "common/threading.h"
+#include "core/core.h"
 #include "os/os_specific.h"
 #include "strings/string_utils.h"
 
@@ -262,11 +263,11 @@ static rdcarray<EnvironmentModification> &GetEnvModifications()
   return envCallbacks;
 }
 
-static std::map<rdcstr, rdcstr> EnvStringToEnvMap(const char **envstring)
+static std::map<rdcstr, rdcstr> EnvStringToEnvMap(char *const *envstring)
 {
   std::map<rdcstr, rdcstr> ret;
 
-  const char **e = envstring;
+  char *const *e = envstring;
 
   while(*e)
   {
@@ -341,6 +342,61 @@ void Process::RegisterEnvironmentModification(const EnvironmentModification &mod
   GetEnvModifications().push_back(modif);
 }
 
+void ApplySingleEnvMod(EnvironmentModification &m, rdcstr &value)
+{
+  switch(m.mod)
+  {
+    case EnvMod::Set: value = m.value.c_str(); break;
+    case EnvMod::Append:
+    {
+      if(!value.empty())
+      {
+        if(m.sep == EnvSep::Platform || m.sep == EnvSep::Colon)
+          value += ":";
+        else if(m.sep == EnvSep::SemiColon)
+          value += ";";
+      }
+      value += m.value.c_str();
+      break;
+    }
+    case EnvMod::Prepend:
+    {
+      if(!value.empty())
+      {
+        rdcstr prep = m.value;
+        if(m.sep == EnvSep::Platform || m.sep == EnvSep::Colon)
+          prep += ":";
+        else if(m.sep == EnvSep::SemiColon)
+          prep += ";";
+        value = prep + value;
+      }
+      else
+      {
+        value = m.value.c_str();
+      }
+      break;
+    }
+  }
+}
+
+void ApplyEnvironmentModifications(rdcarray<EnvironmentModification> &modifications)
+{
+  // turn environment string to a UTF-8 map
+  char **currentEnvironment = GetCurrentEnvironment();
+  std::map<rdcstr, rdcstr> currentEnv = EnvStringToEnvMap(currentEnvironment);
+
+  for(size_t i = 0; i < modifications.size(); i++)
+  {
+    EnvironmentModification &m = modifications[i];
+
+    rdcstr value = currentEnv[m.name.c_str()];
+
+    ApplySingleEnvMod(m, value);
+
+    setenv(m.name.c_str(), value.c_str(), true);
+  }
+}
+
 // on linux we apply environment changes before launching the program, as
 // there is no support for injecting/loading renderdoc into a running program
 // in any way, and we also have some environment changes that we *have* to make
@@ -350,53 +406,8 @@ void Process::RegisterEnvironmentModification(const EnvironmentModification &mod
 // in process (e.g. if we notice a setting and want to enable an env var as a result)
 void Process::ApplyEnvironmentModification()
 {
-  // turn environment string to a UTF-8 map
-  char **currentEnvironment = GetCurrentEnvironment();
-  std::map<rdcstr, rdcstr> currentEnv = EnvStringToEnvMap((const char **)currentEnvironment);
   rdcarray<EnvironmentModification> &modifications = GetEnvModifications();
-
-  for(size_t i = 0; i < modifications.size(); i++)
-  {
-    EnvironmentModification &m = modifications[i];
-
-    rdcstr value = currentEnv[m.name.c_str()];
-
-    switch(m.mod)
-    {
-      case EnvMod::Set: value = m.value.c_str(); break;
-      case EnvMod::Append:
-      {
-        if(!value.empty())
-        {
-          if(m.sep == EnvSep::Platform || m.sep == EnvSep::Colon)
-            value += ":";
-          else if(m.sep == EnvSep::SemiColon)
-            value += ";";
-        }
-        value += m.value.c_str();
-        break;
-      }
-      case EnvMod::Prepend:
-      {
-        if(!value.empty())
-        {
-          rdcstr prep = m.value;
-          if(m.sep == EnvSep::Platform || m.sep == EnvSep::Colon)
-            prep += ":";
-          else if(m.sep == EnvSep::SemiColon)
-            prep += ";";
-          value = prep + value;
-        }
-        else
-        {
-          value = m.value.c_str();
-        }
-        break;
-      }
-    }
-
-    setenv(m.name.c_str(), value.c_str(), true);
-  }
+  ApplyEnvironmentModifications(modifications);
 
   // these have been applied to the current process
   modifications.clear();
@@ -712,25 +723,9 @@ uint32_t Process::LaunchScript(const rdcstr &script, const rdcstr &workingDir,
   return LaunchProcess("bash", workingDir, args, internal, result);
 }
 
-rdcpair<ReplayStatus, uint32_t> Process::LaunchAndInjectIntoProcess(
-    const rdcstr &app, const rdcstr &workingDir, const rdcstr &cmdLine,
-    const rdcarray<EnvironmentModification> &envList, const rdcstr &capturefile,
-    const CaptureOptions &opts, bool waitForExit)
+void GetHookingEnvMods(rdcarray<EnvironmentModification> &modifications, const CaptureOptions &opts,
+                       const rdcstr &capturefile)
 {
-  if(app.empty())
-  {
-    RDCERR("Invalid empty 'app'");
-    return {ReplayStatus::InternalError, 0};
-  }
-
-  // turn environment string to a UTF-8 map
-  char **currentEnvironment = GetCurrentEnvironment();
-  std::map<rdcstr, rdcstr> env = EnvStringToEnvMap((const char **)currentEnvironment);
-  rdcarray<EnvironmentModification> modifications = GetEnvModifications();
-
-  for(const EnvironmentModification &e : envList)
-    modifications.push_back(e);
-
   rdcstr binpath, libpath, ownlibpath;
   {
     FileIO::GetExecutableFilename(binpath);
@@ -759,6 +754,12 @@ rdcpair<ReplayStatus, uint32_t> Process::LaunchAndInjectIntoProcess(
 
   rdcstr optstr = opts.EncodeAsString();
 
+  modifications.push_back(EnvironmentModification(EnvMod::Append, EnvSep::Platform,
+                                                  "RENDERDOC_ORIGLIBPATH",
+                                                  Process::GetEnvVariable(LIB_PATH_ENV_VAR)));
+  modifications.push_back(EnvironmentModification(EnvMod::Append, EnvSep::Platform,
+                                                  "RENDERDOC_ORIGPRELOAD",
+                                                  Process::GetEnvVariable(PRELOAD_ENV_VAR)));
   modifications.push_back(
       EnvironmentModification(EnvMod::Append, EnvSep::Platform, LIB_PATH_ENV_VAR, binpath));
   modifications.push_back(
@@ -773,6 +774,121 @@ rdcpair<ReplayStatus, uint32_t> Process::LaunchAndInjectIntoProcess(
       EnvironmentModification(EnvMod::Set, EnvSep::NoSep, "RENDERDOC_CAPOPTS", optstr));
   modifications.push_back(EnvironmentModification(EnvMod::Set, EnvSep::NoSep,
                                                   "RENDERDOC_DEBUG_LOG_FILE", RDCGETLOGFILE()));
+}
+
+void PreForkConfigureHooks()
+{
+  rdcarray<EnvironmentModification> modifications;
+
+  GetHookingEnvMods(modifications, RenderDoc::Inst().GetCaptureOptions(),
+                    RenderDoc::Inst().GetCaptureFileTemplate());
+
+  ApplyEnvironmentModifications(modifications);
+}
+
+void GetUnhookedEnvp(char *const *envp, rdcstr &envpStr, rdcarray<char *> &modifiedEnv)
+{
+  std::map<rdcstr, rdcstr> envmap = EnvStringToEnvMap(envp);
+
+  // this is a nasty hack. We set this env var when we inject into a child, but because we don't
+  // know when vulkan may be initialised we need to leave it on indefinitely. If we're not
+  // injecting into children we need to unset this variable so it doesn't get inherited.
+  envmap.erase(RENDERDOC_VULKAN_LAYER_VAR);
+
+  envpStr.clear();
+
+  // flatten the map to a string
+  for(auto it = envmap.begin(); it != envmap.end(); it++)
+  {
+    envpStr += it->first;
+    envpStr += "=";
+    envpStr += it->second;
+    envpStr.push_back('\0');
+  }
+  envpStr.push_back('\0');
+
+  // create the array desired
+  char *c = envpStr.data();
+  while(*c)
+  {
+    modifiedEnv.push_back(c);
+    c += strlen(c) + 1;
+  }
+  modifiedEnv.push_back(NULL);
+}
+
+void GetHookedEnvp(char *const *envp, rdcstr &envpStr, rdcarray<char *> &modifiedEnv)
+{
+  rdcarray<EnvironmentModification> modifications;
+
+  GetHookingEnvMods(modifications, RenderDoc::Inst().GetCaptureOptions(),
+                    RenderDoc::Inst().GetCaptureFileTemplate());
+
+  std::map<rdcstr, rdcstr> envmap = EnvStringToEnvMap(envp);
+
+  for(EnvironmentModification &mod : modifications)
+  {
+    // update the values for original values we're storing, since they were gotten by querying the
+    // *current* environment not envp here.
+    if(mod.name == "RENDERDOC_ORIGLIBPATH")
+      mod.value = envmap[LIB_PATH_ENV_VAR];
+    else if(mod.name == "RENDERDOC_ORIGPRELOAD")
+      mod.value = envmap[PRELOAD_ENV_VAR];
+
+    // modify the map in-place
+    ApplySingleEnvMod(mod, envmap[mod.name.c_str()]);
+  }
+
+  envpStr.clear();
+
+  // flatten the map to a string
+  for(auto it = envmap.begin(); it != envmap.end(); it++)
+  {
+    envpStr += it->first;
+    envpStr += "=";
+    envpStr += it->second;
+    envpStr.push_back('\0');
+  }
+  envpStr.push_back('\0');
+
+  // create the array desired
+  char *c = envpStr.data();
+  while(*c)
+  {
+    modifiedEnv.push_back(c);
+    c += strlen(c) + 1;
+  }
+  modifiedEnv.push_back(NULL);
+}
+
+void ResetHookingEnvVars()
+{
+  setenv(LIB_PATH_ENV_VAR, Process::GetEnvVariable("RENDERDOC_ORIGLIBPATH").c_str(), true);
+  setenv(PRELOAD_ENV_VAR, Process::GetEnvVariable("RENDERDOC_ORIGPRELOAD").c_str(), true);
+  unsetenv("RENDERDOC_ORIGLIBPATH");
+  unsetenv("RENDERDOC_ORIGPRELOAD");
+}
+
+rdcpair<ReplayStatus, uint32_t> Process::LaunchAndInjectIntoProcess(
+    const rdcstr &app, const rdcstr &workingDir, const rdcstr &cmdLine,
+    const rdcarray<EnvironmentModification> &envList, const rdcstr &capturefile,
+    const CaptureOptions &opts, bool waitForExit)
+{
+  if(app.empty())
+  {
+    RDCERR("Invalid empty 'app'");
+    return {ReplayStatus::InternalError, 0};
+  }
+
+  // turn environment string to a UTF-8 map
+  char **currentEnvironment = GetCurrentEnvironment();
+  std::map<rdcstr, rdcstr> env = EnvStringToEnvMap(currentEnvironment);
+  rdcarray<EnvironmentModification> modifications = GetEnvModifications();
+
+  for(const EnvironmentModification &e : envList)
+    modifications.push_back(e);
+
+  GetHookingEnvMods(modifications, opts, capturefile);
 
   for(size_t i = 0; i < modifications.size(); i++)
   {

renderdoc/renderdoc.version

@@ -8,6 +8,9 @@
                 dlopen;
                 dlsym;
                 fork;
+                execle;
+                execve;
+                execvpe;
                 _exit;
                 RENDERDOC_*;
                 VK_LAYER_RENDERDOC_*;