Register child processes on linux by intercepting fork. Closes #1883

renderdoc/core/core.cpp

@@ -825,6 +825,27 @@ void RenderDoc::Tick()
 
   prev_focus = cur_focus;
   prev_cap = cur_cap;
+
+  // check for any child threads that need to be waited on, remove them from the list
+  rdcarray<Threading::ThreadHandle> waitThreads;
+  {
+    SCOPED_LOCK(m_ChildLock);
+    for(rdcpair<uint32_t, Threading::ThreadHandle> &c : m_ChildThreads)
+    {
+      if(c.first == 0)
+        waitThreads.push_back(c.second);
+    }
+
+    m_ChildThreads.removeIf(
+        [](const rdcpair<uint32_t, Threading::ThreadHandle> &c) { return c.first == 0; });
+  }
+
+  // clean up the threads now
+  for(Threading::ThreadHandle t : waitThreads)
+  {
+    Threading::JoinThread(t);
+    Threading::CloseThread(t);
+  }
 }
 
 void RenderDoc::CycleActiveWindow()
@@ -1684,12 +1705,29 @@ void RenderDoc::AddChildProcess(uint32_t pid, uint32_t ident)
   m_Children.push_back(make_rdcpair(pid, ident));
 }
 
-rdcarray<rdcpair<uint32_t, uint32_t> > RenderDoc::GetChildProcesses()
+rdcarray<rdcpair<uint32_t, uint32_t>> RenderDoc::GetChildProcesses()
 {
   SCOPED_LOCK(m_ChildLock);
   return m_Children;
 }
 
+void RenderDoc::CompleteChildThread(uint32_t pid)
+{
+  SCOPED_LOCK(m_ChildLock);
+  // the thread for this PID is done, mark it as ready to wait on by zero-ing out the PID
+  for(rdcpair<uint32_t, Threading::ThreadHandle> &c : m_ChildThreads)
+  {
+    if(c.first == pid)
+      c.first = 0;
+  }
+}
+
+void RenderDoc::AddChildThread(uint32_t pid, Threading::ThreadHandle thread)
+{
+  SCOPED_LOCK(m_ChildLock);
+  m_ChildThreads.push_back(make_rdcpair(pid, thread));
+}
+
 rdcarray<CaptureData> RenderDoc::GetCaptures()
 {
   SCOPED_LOCK(m_CaptureLock);

renderdoc/core/core.h

@@ -446,7 +446,10 @@ public:
   void FinishCaptureWriting(RDCFile *rdc, uint32_t frameNumber);
 
   void AddChildProcess(uint32_t pid, uint32_t ident);
-  rdcarray<rdcpair<uint32_t, uint32_t> > GetChildProcesses();
+  rdcarray<rdcpair<uint32_t, uint32_t>> GetChildProcesses();
+
+  void CompleteChildThread(uint32_t pid);
+  void AddChildThread(uint32_t pid, Threading::ThreadHandle thread);
 
   rdcarray<CaptureData> GetCaptures();
 
@@ -624,7 +627,8 @@ private:
   rdcarray<CaptureData> m_Captures;
 
   Threading::CriticalSection m_ChildLock;
-  rdcarray<rdcpair<uint32_t, uint32_t> > m_Children;
+  rdcarray<rdcpair<uint32_t, uint32_t>> m_Children;
+  rdcarray<rdcpair<uint32_t, Threading::ThreadHandle>> m_ChildThreads;
 
   std::map<RDCDriver, ReplayDriverProvider> m_ReplayDriverProviders;
   std::map<RDCDriver, RemoteDriverProvider> m_RemoteDriverProviders;

renderdoc/os/posix/linux/linux_hook.cpp

@@ -24,9 +24,12 @@
 
 #include <dlfcn.h>
 #include <string.h>
+#include <sys/types.h>
+#include <unistd.h>
 #include <algorithm>
 #include <map>
 #include "common/threading.h"
+#include "core/core.h"
 #include "hooks/hooks.h"
 #include "os/os_specific.h"
 #include "plthook/plthook.h"
@@ -41,9 +44,11 @@ static rdcarray<FunctionHook> functionHooks;
 void *intercept_dlopen(const char *filename, int flag, void *ret);
 void plthook_lib(void *handle);
 
+typedef pid_t (*FORKPROC)();
 typedef void *(*DLOPENPROC)(const char *, int);
 typedef void *(*DLSYMPROC)(void *, const char *);
 DLOPENPROC realdlopen = NULL;
+FORKPROC realfork = NULL;
 DLSYMPROC realdlsym = NULL;
 
 static volatile int32_t tlsbusyflag = 0;
@@ -76,6 +81,45 @@ __attribute__((visibility("default"))) void *dlopen(const char *filename, int fl
   return ret;
 }
 
+int GetIdentPort(pid_t childPid);
+
+__attribute__((visibility("default"))) pid_t fork()
+{
+  if(!realfork)
+  {
+    FORKPROC passthru = (FORKPROC)dlsym(RTLD_NEXT, "fork");
+
+    return passthru();
+  }
+
+  // fork in a captured application. Need to get the child ident and register it
+
+  pid_t ret = realfork();
+
+  if(ret > 0)
+  {
+    // in parent process, kick off a thread to get the ident
+    Threading::ThreadHandle handle = Threading::CreateThread([ret]() {
+      // don't accept a return value of our own ident, that means we've checked too early and exec
+      // hasn't run yet
+      const uint32_t ownIdent = RenderDoc::Inst().GetTargetControlIdent();
+      uint32_t ident = ownIdent;
+      for(uint32_t i = 0; i < 10 && ident == ownIdent; i++)
+      {
+        ident = (uint32_t)GetIdentPort(ret);
+        if(ident == ownIdent)
+          usleep(1000);
+      }
+
+      RenderDoc::Inst().AddChildProcess((uint32_t)ret, (uint32_t)ident);
+      RenderDoc::Inst().CompleteChildThread((uint32_t)ret);
+    });
+    RenderDoc::Inst().AddChildThread((uint32_t)ret, handle);
+  }
+
+  return ret;
+}
+
 #if defined(RENDERDOC_HOOK_DLSYM)
 
 #pragma message("ALERT: dlsym() hooking enabled! This is unreliable & relies on glibc internals.")
@@ -223,6 +267,7 @@ void *intercept_dlopen(const char *filename, int flag, void *ret)
 void LibraryHooks::BeginHookRegistration()
 {
   realdlopen = (DLOPENPROC)dlsym(RTLD_NEXT, "dlopen");
+  realfork = (FORKPROC)dlsym(RTLD_NEXT, "fork");
 }
 
 bool LibraryHooks::Detect(const char *identifier)

renderdoc/renderdoc.version

@@ -7,6 +7,7 @@
                 vk_icd*;
                 dlopen;
                 dlsym;
+                fork;
                 _exit;
                 RENDERDOC_*;
                 VK_LAYER_RENDERDOC_*;