Try to handle processes on linux that fork without exec

* This is common since bash does it, so running a bash script will likely hit
  it.

renderdoc/os/posix/android/android_process.cpp

@@ -102,7 +102,7 @@ void StopAtMainInChild()
 {
 }
 
-bool StopChildAtMain(pid_t childPid)
+bool StopChildAtMain(pid_t childPid, bool *exitWithNoExec)
 {
   return false;
 }

renderdoc/os/posix/apple/apple_process.cpp

@@ -154,7 +154,7 @@ void StopAtMainInChild()
 {
 }
 
-bool StopChildAtMain(pid_t childPid)
+bool StopChildAtMain(pid_t childPid, bool *exitWithNoExec)
 {
   return false;
 }

renderdoc/os/posix/ggp/ggp_process.cpp

@@ -140,7 +140,7 @@ void StopAtMainInChild()
 {
 }
 
-bool StopChildAtMain(pid_t childPid)
+bool StopChildAtMain(pid_t childPid, bool *exitWithNoExec)
 {
   return false;
 }

renderdoc/os/posix/linux/linux_hook.cpp

@@ -100,7 +100,7 @@ void GetUnhookedEnvp(char *const *envp, rdcstr &envpStr, rdcarray<char *> &modif
 void GetHookedEnvp(char *const *envp, rdcstr &envpStr, rdcarray<char *> &modifiedEnv);
 void ResetHookingEnvVars();
 void StopAtMainInChild();
-bool StopChildAtMain(pid_t childPid);
+bool StopChildAtMain(pid_t childPid, bool *exitWithNoExec);
 void ResumeProcess(pid_t childPid, uint32_t delay = 0);
 int direct_setenv(const char *name, const char *value, int overwrite);
 
@@ -324,9 +324,15 @@ __attribute__((visibility("default"))) pid_t fork()
     if(Linux_Debug_PtraceLogging())
       RDCLOG("hooked fork() in parent, child is %d", ret);
 
-    bool stopped = StopChildAtMain(ret);
+    bool exitWithNoExec = false;
+    bool stopped = StopChildAtMain(ret, &exitWithNoExec);
 
-    if(stopped)
+    if(exitWithNoExec)
+    {
+      if(Linux_Debug_PtraceLogging())
+        RDCLOG("hooked fork() child %d exited gracefully while waiting for exec(). Ignoring", ret);
+    }
+    else if(stopped)
     {
       int ident = GetIdentPort(ret);
 
@@ -376,6 +382,9 @@ __attribute__((visibility("default"))) pid_t fork()
     }
   }
 
+  if(Linux_Debug_PtraceLogging())
+    RDCLOG("Returning from fork");
+
   return ret;
 }
 

renderdoc/os/posix/linux/linux_process.cpp

@@ -24,6 +24,7 @@
 
 #include <dlfcn.h>
 #include <elf.h>
+#include <errno.h>
 #include <stdlib.h>
 #include <sys/ptrace.h>
 #include <sys/types.h>
@@ -91,6 +92,7 @@ int GetIdentPort(pid_t childPid)
 {
   int ret = 0;
 
+  rdcstr pidvalidfile = StringFormat::Fmt("/proc/%d/stat", (int)childPid);
   rdcstr procfile = StringFormat::Fmt("/proc/%d/net/tcp", (int)childPid);
 
   int waitTime = INITIAL_WAIT_TIME;
@@ -98,6 +100,13 @@ int GetIdentPort(pid_t childPid)
   // try for a little while for the /proc entry to appear
   while(ret == 0 && waitTime <= MAX_WAIT_TIME)
   {
+    if(!FileIO::exists(pidvalidfile))
+    {
+      RDCWARN("Process %u is not running - did it exit during initialisation or fail to run?",
+              childPid);
+      return 0;
+    }
+
     // back-off for each retry
     usleep(waitTime);
 
@@ -288,7 +297,7 @@ static bool wait_traced_child(pid_t childPid, uint32_t timeoutMS, int &status)
   return WIFSTOPPED(status);
 }
 
-bool StopChildAtMain(pid_t childPid)
+bool StopChildAtMain(pid_t childPid, bool *exitWithNoExec)
 {
   // don't do this unless the ptrace scope is OK.
   if(!ptrace_scope_ok())
@@ -320,7 +329,7 @@ bool StopChildAtMain(pid_t childPid)
   int64_t ptraceRet = 0;
 
   // continue until exec
-  ptraceRet = ptrace(PTRACE_SETOPTIONS, childPid, NULL, PTRACE_O_TRACEEXEC);
+  ptraceRet = ptrace(PTRACE_SETOPTIONS, childPid, NULL, PTRACE_O_TRACEEXEC | PTRACE_O_TRACEEXIT);
   RDCASSERTEQUAL(ptraceRet, 0);
 
   if(Linux_Debug_PtraceLogging())
@@ -338,15 +347,33 @@ bool StopChildAtMain(pid_t childPid)
     return false;
   }
 
-  if(childStatus > 0 && (childStatus >> 8) != (SIGTRAP | (PTRACE_EVENT_EXEC << 8)))
+  int statusResult = childStatus >> 8;
+
+  if(childStatus > 0 &&
+     (statusResult == SIGCHLD || statusResult == (SIGTRAP | (PTRACE_EVENT_EXIT << 8))))
   {
-    RDCERR("Exec wait event from child PID %u was status %x, expected %x", childPid,
-           (childStatus >> 8), (SIGTRAP | (PTRACE_EVENT_EXEC << 8)));
+    if(Linux_Debug_PtraceLogging())
+      RDCLOG("Child PID %u exited while waiting for exec() 0x%x", childPid, childStatus);
+    if(exitWithNoExec)
+      *exitWithNoExec = true;
+
+    if(statusResult == SIGCHLD)
+      ptrace(PTRACE_DETACH, childPid, NULL, SIGCHLD);
+    else
+      ptrace(PTRACE_DETACH, childPid, NULL, NULL);
+    return false;
+  }
+
+  if(childStatus > 0 && statusResult != (SIGTRAP | (PTRACE_EVENT_EXEC << 8)))
+  {
+    RDCERR("Exec wait event from child PID %u was status 0x%x, expected 0x%x", childPid,
+           statusResult, (SIGTRAP | (PTRACE_EVENT_EXEC << 8)));
+
     return false;
   }
 
   if(Linux_Debug_PtraceLogging())
-    RDCLOG("Child PID %u is stopped at execve()", childPid);
+    RDCLOG("Child PID %u is stopped at execve() 0x%x", childPid, childStatus);
 
   rdcstr exepath;
   long baseVirtualPointer = 0;
@@ -551,6 +578,9 @@ void StopAtMainInChild()
 
 void ResumeProcess(pid_t childPid, uint32_t delaySeconds)
 {
+  if(!ptrace_scope_ok())
+    return;
+
   if(childPid != 0)
   {
     // if we have a delay, see if the process is paused. If so then detach it but keep it stopped
@@ -561,6 +591,10 @@ void ResumeProcess(pid_t childPid, uint32_t delaySeconds)
 
       if(ip != 0)
       {
+        if(Linux_Debug_PtraceLogging())
+          RDCLOG("Detaching %u with SIGSTOP to allow a debugger to attach, waiting %u seconds",
+                 childPid, delaySeconds);
+
         // detach but stop, to allow a debugger to attach
         ptrace(PTRACE_DETACH, childPid, NULL, SIGSTOP);
 
@@ -613,8 +647,14 @@ void ResumeProcess(pid_t childPid, uint32_t delaySeconds)
       }
     }
 
+    if(Linux_Debug_PtraceLogging())
+      RDCLOG("Detaching immediately from %u", childPid);
+
     // try to detach and resume the process, ignoring any errors if we weren't tracing
-    ptrace(PTRACE_DETACH, childPid, NULL, NULL);
+    long ret = ptrace(PTRACE_DETACH, childPid, NULL, NULL);
+
+    if(Linux_Debug_PtraceLogging())
+      RDCLOG("Detached pid %u (%ld)", childPid, ret);
   }
 }
 

renderdoc/os/posix/posix_process.cpp

@@ -52,7 +52,7 @@ int GetIdentPort(pid_t childPid);
 // functions to try and let the child run just far enough to get to main() but no further. This lets
 // us check the ident port and resume.
 void StopAtMainInChild();
-bool StopChildAtMain(pid_t childPid);
+bool StopChildAtMain(pid_t childPid, bool *exitWithNoExec);
 void ResumeProcess(pid_t childPid, uint32_t delay = 0);
 
 #if ENABLED(RDOC_APPLE)
@@ -634,7 +634,7 @@ static pid_t RunProcess(rdcstr appName, rdcstr workDir, const rdcstr &cmdLine, c
     else
     {
       if(pauseAtMain)
-        StopChildAtMain(childPid);
+        StopChildAtMain(childPid, NULL);
 
       if(!stdoutPipe)
       {