From 1464eaeb81dfc0b14b5f2e100560c8fd51d9ae7f Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Wed, 6 Oct 2021 16:26:18 +0100
Subject: [PATCH] Ensure linux process hooks don't do anything on replay

---
 renderdoc/hooks/hooks.h                     |  4 ++++
 renderdoc/os/posix/android/android_hook.cpp |  5 +++++
 renderdoc/os/posix/apple/apple_hook.cpp     |  5 +++++
 renderdoc/os/posix/ggp/ggp_hook.cpp         |  4 ++++
 renderdoc/os/posix/linux/linux_hook.cpp     | 21 +++++++++++++++++++++
 renderdoc/os/posix/posix_libentry.cpp       |  2 ++
 renderdoc/os/win32/win32_hook.cpp           |  4 ++++
 renderdoc/os/win32/win32_libentry.cpp       |  2 ++
 8 files changed, 47 insertions(+)

diff --git a/renderdoc/hooks/hooks.h b/renderdoc/hooks/hooks.h
index 1ca719e90..b3305506e 100644
--- a/renderdoc/hooks/hooks.h
+++ b/renderdoc/hooks/hooks.h
@@ -151,6 +151,10 @@ public:
 
   // platform specific implementations
 
+  // some platforms may unavoidably hook on replay, this gives them a chance to do any
+  // initialisation needed to ensure those hooks don't do anything
+  static void ReplayInitialise();
+
   // Removes hooks (where possible) and restores everything to an un-hooked state
   static void RemoveHooks();
 
diff --git a/renderdoc/os/posix/android/android_hook.cpp b/renderdoc/os/posix/android/android_hook.cpp
index 056f79292..464db1f72 100644
--- a/renderdoc/os/posix/android/android_hook.cpp
+++ b/renderdoc/os/posix/android/android_hook.cpp
@@ -685,6 +685,11 @@ void LibraryHooks::RemoveHooks()
   RDCERR("Removing hooks is not possible on this platform");
 }
 
+void LibraryHooks::ReplayInitialise()
+{
+  // nothing to do
+}
+
 void LibraryHooks::BeginHookRegistration()
 {
   // nothing to do
diff --git a/renderdoc/os/posix/apple/apple_hook.cpp b/renderdoc/os/posix/apple/apple_hook.cpp
index 6030ae734..522e69c61 100644
--- a/renderdoc/os/posix/apple/apple_hook.cpp
+++ b/renderdoc/os/posix/apple/apple_hook.cpp
@@ -99,6 +99,11 @@ void LibraryHooks::RemoveHooks()
   RDCERR("Removing hooks is not possible on this platform");
 }
 
+void LibraryHooks::ReplayInitialise()
+{
+  // nothing to do
+}
+
 void LibraryHooks::EndHookRegistration()
 {
   // process libraries with callbacks by loading them if necessary (though we should be linked to
diff --git a/renderdoc/os/posix/ggp/ggp_hook.cpp b/renderdoc/os/posix/ggp/ggp_hook.cpp
index ad044f872..7da413178 100644
--- a/renderdoc/os/posix/ggp/ggp_hook.cpp
+++ b/renderdoc/os/posix/ggp/ggp_hook.cpp
@@ -43,6 +43,10 @@ void LibraryHooks::RemoveHooks()
 {
 }
 
+void LibraryHooks::ReplayInitialise()
+{
+}
+
 void LibraryHooks::Refresh()
 {
 }
diff --git a/renderdoc/os/posix/linux/linux_hook.cpp b/renderdoc/os/posix/linux/linux_hook.cpp
index 4989e2865..4e8e5382b 100644
--- a/renderdoc/os/posix/linux/linux_hook.cpp
+++ b/renderdoc/os/posix/linux/linux_hook.cpp
@@ -76,6 +76,9 @@ __attribute__((visibility("default"))) void *dlopen(const char *filename, int fl
     return ret;
   }
 
+  if(RenderDoc::Inst().IsReplayApp())
+    return realdlopen(filename, flag);
+
   // don't do any hook processing inside here even if we call dlopen again
   Atomic::Inc32(&tlsbusyflag);
   void *ret = realdlopen(filename, flag);
@@ -203,6 +206,9 @@ __attribute__((visibility("default"))) int execve(const char *pathname, char *co
     return passthru(pathname, argv, envp);
   }
 
+  if(RenderDoc::Inst().IsReplayApp())
+    return realexecve(pathname, argv, envp);
+
   rdcarray<char *> modifiedEnv;
   rdcstr envpStr;
 
@@ -236,6 +242,9 @@ __attribute__((visibility("default"))) int execvpe(const char *pathname, char *c
     return passthru(pathname, argv, envp);
   }
 
+  if(RenderDoc::Inst().IsReplayApp())
+    return realexecvpe(pathname, argv, envp);
+
   rdcarray<char *> modifiedEnv;
   rdcstr envpStr;
 
@@ -265,6 +274,9 @@ __attribute__((visibility("default"))) pid_t fork()
     return passthru();
   }
 
+  if(RenderDoc::Inst().IsReplayApp())
+    return realfork();
+
   // if we're not hooking children just call to the real one, we don't have to do anything
   if(!RenderDoc::Inst().GetCaptureOptions().hookIntoChildren)
   {
@@ -510,6 +522,15 @@ void *intercept_dlopen(const char *filename, int flag, void *ret)
   return ret;
 }
 
+void LibraryHooks::ReplayInitialise()
+{
+  realdlopen = (DLOPENPROC)dlsym(RTLD_NEXT, "dlopen");
+  realfork = (FORKPROC)dlsym(RTLD_NEXT, "fork");
+  realexecle = (EXECLEPROC)dlsym(RTLD_NEXT, "execle");
+  realexecve = (EXECVEPROC)dlsym(RTLD_NEXT, "execve");
+  realexecvpe = (EXECVPEPROC)dlsym(RTLD_NEXT, "execvpe");
+}
+
 void LibraryHooks::BeginHookRegistration()
 {
   realdlopen = (DLOPENPROC)dlsym(RTLD_NEXT, "dlopen");
diff --git a/renderdoc/os/posix/posix_libentry.cpp b/renderdoc/os/posix/posix_libentry.cpp
index 1cac38b08..a957e65aa 100644
--- a/renderdoc/os/posix/posix_libentry.cpp
+++ b/renderdoc/os/posix/posix_libentry.cpp
@@ -39,6 +39,8 @@ void library_loaded()
 
     RenderDoc::Inst().Initialise();
 
+    LibraryHooks::ReplayInitialise();
+
     return;
   }
   else
diff --git a/renderdoc/os/win32/win32_hook.cpp b/renderdoc/os/win32/win32_hook.cpp
index 9184d3b4e..aac5ca664 100644
--- a/renderdoc/os/win32/win32_hook.cpp
+++ b/renderdoc/os/win32/win32_hook.cpp
@@ -940,6 +940,10 @@ void LibraryHooks::Refresh()
   // don't need to refresh on windows
 }
 
+void LibraryHooks::ReplayInitialise()
+{
+}
+
 void LibraryHooks::RemoveHooks()
 {
   LibraryHooks::RemoveHookCallbacks();
diff --git a/renderdoc/os/win32/win32_libentry.cpp b/renderdoc/os/win32/win32_libentry.cpp
index 2d2919d3d..88a8a79ad 100644
--- a/renderdoc/os/win32/win32_libentry.cpp
+++ b/renderdoc/os/win32/win32_libentry.cpp
@@ -58,6 +58,8 @@ static BOOL add_hooks()
 
     RenderDoc::Inst().Initialise();
 
+    LibraryHooks::ReplayInitialise();
+
     return true;
   }