Ensure linux process hooks don't do anything on replay

renderdoc/hooks/hooks.h

@@ -151,6 +151,10 @@ public:
 
   // platform specific implementations
 
+  // some platforms may unavoidably hook on replay, this gives them a chance to do any
+  // initialisation needed to ensure those hooks don't do anything
+  static void ReplayInitialise();
+
   // Removes hooks (where possible) and restores everything to an un-hooked state
   static void RemoveHooks();
 

renderdoc/os/posix/android/android_hook.cpp

@@ -685,6 +685,11 @@ void LibraryHooks::RemoveHooks()
   RDCERR("Removing hooks is not possible on this platform");
 }
 
+void LibraryHooks::ReplayInitialise()
+{
+  // nothing to do
+}
+
 void LibraryHooks::BeginHookRegistration()
 {
   // nothing to do

renderdoc/os/posix/apple/apple_hook.cpp

@@ -99,6 +99,11 @@ void LibraryHooks::RemoveHooks()
   RDCERR("Removing hooks is not possible on this platform");
 }
 
+void LibraryHooks::ReplayInitialise()
+{
+  // nothing to do
+}
+
 void LibraryHooks::EndHookRegistration()
 {
   // process libraries with callbacks by loading them if necessary (though we should be linked to

renderdoc/os/posix/ggp/ggp_hook.cpp

@@ -43,6 +43,10 @@ void LibraryHooks::RemoveHooks()
 {
 }
 
+void LibraryHooks::ReplayInitialise()
+{
+}
+
 void LibraryHooks::Refresh()
 {
 }

renderdoc/os/posix/linux/linux_hook.cpp

@@ -76,6 +76,9 @@ __attribute__((visibility("default"))) void *dlopen(const char *filename, int fl
     return ret;
   }
 
+  if(RenderDoc::Inst().IsReplayApp())
+    return realdlopen(filename, flag);
+
   // don't do any hook processing inside here even if we call dlopen again
   Atomic::Inc32(&tlsbusyflag);
   void *ret = realdlopen(filename, flag);
@@ -203,6 +206,9 @@ __attribute__((visibility("default"))) int execve(const char *pathname, char *co
     return passthru(pathname, argv, envp);
   }
 
+  if(RenderDoc::Inst().IsReplayApp())
+    return realexecve(pathname, argv, envp);
+
   rdcarray<char *> modifiedEnv;
   rdcstr envpStr;
 
@@ -236,6 +242,9 @@ __attribute__((visibility("default"))) int execvpe(const char *pathname, char *c
     return passthru(pathname, argv, envp);
   }
 
+  if(RenderDoc::Inst().IsReplayApp())
+    return realexecvpe(pathname, argv, envp);
+
   rdcarray<char *> modifiedEnv;
   rdcstr envpStr;
 
@@ -265,6 +274,9 @@ __attribute__((visibility("default"))) pid_t fork()
     return passthru();
   }
 
+  if(RenderDoc::Inst().IsReplayApp())
+    return realfork();
+
   // if we're not hooking children just call to the real one, we don't have to do anything
   if(!RenderDoc::Inst().GetCaptureOptions().hookIntoChildren)
   {
@@ -510,6 +522,15 @@ void *intercept_dlopen(const char *filename, int flag, void *ret)
   return ret;
 }
 
+void LibraryHooks::ReplayInitialise()
+{
+  realdlopen = (DLOPENPROC)dlsym(RTLD_NEXT, "dlopen");
+  realfork = (FORKPROC)dlsym(RTLD_NEXT, "fork");
+  realexecle = (EXECLEPROC)dlsym(RTLD_NEXT, "execle");
+  realexecve = (EXECVEPROC)dlsym(RTLD_NEXT, "execve");
+  realexecvpe = (EXECVPEPROC)dlsym(RTLD_NEXT, "execvpe");
+}
+
 void LibraryHooks::BeginHookRegistration()
 {
   realdlopen = (DLOPENPROC)dlsym(RTLD_NEXT, "dlopen");

renderdoc/os/posix/posix_libentry.cpp

@@ -39,6 +39,8 @@ void library_loaded()
 
     RenderDoc::Inst().Initialise();
 
+    LibraryHooks::ReplayInitialise();
+
     return;
   }
   else

renderdoc/os/win32/win32_hook.cpp

@@ -940,6 +940,10 @@ void LibraryHooks::Refresh()
   // don't need to refresh on windows
 }
 
+void LibraryHooks::ReplayInitialise()
+{
+}
+
 void LibraryHooks::RemoveHooks()
 {
   LibraryHooks::RemoveHookCallbacks();

renderdoc/os/win32/win32_libentry.cpp

@@ -58,6 +58,8 @@ static BOOL add_hooks()
 
     RenderDoc::Inst().Initialise();
 
+    LibraryHooks::ReplayInitialise();
+
     return true;
   }