Return ReplayStatus when launching/injecting, use to indicate JDWP error

* If android studio or other android tools are open when a package is
  launched for debugging, they greedily jump on it and connect which
  prevents us from connecting.

qrenderdoc/Code/Interface/QRDInterface.h

@@ -714,14 +714,15 @@ This happens either locally, or on the remote server, depending on whether a con
   the program.
 :param str capturefile: The location to save any captures, if running locally.
 :param CaptureOptions opts: The capture options to use when injecting into the program.
-:return: The ident where the new application is listening for target control, or 0 if something went
-  wrong.
-:rtype: ``int``
+:return: The :class:`ExecuteResult` indicating both the status of the operation (success or failure)
+  and any reason for failure, or else the ident where the new application is listening for target
+  control if everything succeeded.
+:rtype: ExecuteResult
 )");
-  virtual uint32_t ExecuteAndInject(const rdcstr &exe, const rdcstr &workingDir,
-                                    const rdcstr &cmdLine,
-                                    const rdcarray<EnvironmentModification> &env,
-                                    const rdcstr &capturefile, CaptureOptions opts) = 0;
+  virtual ExecuteResult ExecuteAndInject(const rdcstr &exe, const rdcstr &workingDir,
+                                         const rdcstr &cmdLine,
+                                         const rdcarray<EnvironmentModification> &env,
+                                         const rdcstr &capturefile, CaptureOptions opts) = 0;
 
   DOCUMENT(R"(Retrieve a list of drivers that the current remote server supports.
 

qrenderdoc/Code/ReplayManager.cpp

@@ -372,12 +372,12 @@ void ReplayManager::ReopenCaptureFile(const QString &path)
   m_CaptureFile->OpenFile(path.toUtf8().data(), "rdc", NULL);
 }
 
-uint32_t ReplayManager::ExecuteAndInject(const rdcstr &exe, const rdcstr &workingDir,
-                                         const rdcstr &cmdLine,
-                                         const rdcarray<EnvironmentModification> &env,
-                                         const rdcstr &capturefile, CaptureOptions opts)
+ExecuteResult ReplayManager::ExecuteAndInject(const rdcstr &exe, const rdcstr &workingDir,
+                                              const rdcstr &cmdLine,
+                                              const rdcarray<EnvironmentModification> &env,
+                                              const rdcstr &capturefile, CaptureOptions opts)
 {
-  uint32_t ret = 0;
+  ExecuteResult ret;
 
   if(m_Remote)
   {

qrenderdoc/Code/ReplayManager.h

@@ -85,9 +85,9 @@ public:
   ICaptureFile *GetCaptureFile() { return m_CaptureFile; }
   void ReopenCaptureFile(const QString &path);
   const RemoteHost *CurrentRemote() { return m_RemoteHost; }
-  uint32_t ExecuteAndInject(const rdcstr &exe, const rdcstr &workingDir, const rdcstr &cmdLine,
-                            const rdcarray<EnvironmentModification> &env, const rdcstr &capturefile,
-                            CaptureOptions opts);
+  ExecuteResult ExecuteAndInject(const rdcstr &exe, const rdcstr &workingDir, const rdcstr &cmdLine,
+                                 const rdcarray<EnvironmentModification> &env,
+                                 const rdcstr &capturefile, CaptureOptions opts);
 
   rdcarray<rdcstr> GetRemoteSupport();
   void GetHomeFolder(bool synchronous, DirectoryBrowseCallback cb);

qrenderdoc/Windows/MainWindow.cpp

@@ -504,22 +504,38 @@ void MainWindow::OnCaptureTrigger(const QString &exe, const QString &workingDir,
   LambdaThread *th = new LambdaThread([this, exe, workingDir, cmdLine, env, opts, callback]() {
     QString capturefile = m_Ctx.TempCaptureFilename(QFileInfo(exe).baseName());
 
-    uint32_t ret = m_Ctx.Replay().ExecuteAndInject(exe, workingDir, cmdLine, env, capturefile, opts);
+    ExecuteResult ret =
+        m_Ctx.Replay().ExecuteAndInject(exe, workingDir, cmdLine, env, capturefile, opts);
 
     GUIInvoke::call([this, exe, ret, callback]() {
-      if(ret == 0)
+
+      if(ret.status == ReplayStatus::JDWPFailure)
+      {
+        RDDialog::critical(
+            this, tr("Error connecting to debugger"),
+            tr("Error launching %1 for capture.\n\n"
+               "Something went wrong connecting to the debugger on the Android device.\n\n"
+               "This can happen if the package is not "
+               "marked as debuggable, or if another android tool such as android studio "
+               "or other tool is interfering with the debug connection.")
+                .arg(exe));
+        return;
+      }
+
+      if(ret.status != ReplayStatus::Succeeded)
       {
         RDDialog::critical(this, tr("Error kicking capture"),
-                           tr("Error launching %1 for capture.\n\nCheck diagnostic log in Help "
-                              "menu for more details.")
-                               .arg(exe));
+                           tr("Error launching %1 for capture.\n\n%2. Check the diagnostic log in "
+                              "the help menu for more details")
+                               .arg(exe)
+                               .arg(ToQStr(ret.status)));
         return;
       }
 
       LiveCapture *live = new LiveCapture(
           m_Ctx, m_Ctx.Replay().CurrentRemote() ? m_Ctx.Replay().CurrentRemote()->hostname : "",
-          m_Ctx.Replay().CurrentRemote() ? m_Ctx.Replay().CurrentRemote()->Name() : "", ret, this,
-          this);
+          m_Ctx.Replay().CurrentRemote() ? m_Ctx.Replay().CurrentRemote()->Name() : "", ret.ident,
+          this, this);
       ShowLiveCapture(live);
       callback(live);
     });
@@ -546,20 +562,36 @@ void MainWindow::OnInjectTrigger(uint32_t PID, const rdcarray<EnvironmentModific
   LambdaThread *th = new LambdaThread([this, PID, env, name, opts, callback]() {
     QString capturefile = m_Ctx.TempCaptureFilename(name);
 
-    uint32_t ret = RENDERDOC_InjectIntoProcess(PID, env, capturefile.toUtf8().data(), opts, false);
+    ExecuteResult ret =
+        RENDERDOC_InjectIntoProcess(PID, env, capturefile.toUtf8().data(), opts, false);
 
     GUIInvoke::call([this, PID, ret, callback]() {
-      if(ret == 0)
+
+      if(ret.status == ReplayStatus::JDWPFailure)
       {
         RDDialog::critical(
-            this, tr("Error kicking capture"),
-            tr("Error injecting into process %1 for capture.\n\nCheck diagnostic log in "
-               "Help menu for more details.")
+            this, tr("Error connecting to debugger"),
+            tr("Error injecting into process %1 for capture.\n\n"
+               "Something went wrong connecting to the debugger on the Android device.\n\n"
+               "This can happen if the package is not "
+               "marked as debuggable, or if another android tool such as android studio "
+               "or other tool is interfering with the debug connection.")
                 .arg(PID));
         return;
       }
 
-      LiveCapture *live = new LiveCapture(m_Ctx, QString(), QString(), ret, this, this);
+      if(ret.status != ReplayStatus::Succeeded)
+      {
+        RDDialog::critical(
+            this, tr("Error kicking capture"),
+            tr("Error injecting into process %1 for capture.\n\n%2. Check the diagnostic log in "
+               "the help menu for more details")
+                .arg(PID)
+                .arg(ToQStr(ret.status)));
+        return;
+      }
+
+      LiveCapture *live = new LiveCapture(m_Ctx, QString(), QString(), ret.ident, this, this);
       ShowLiveCapture(live);
     });
   });

renderdoc/android/android.cpp

@@ -122,13 +122,17 @@ int GetCurrentPID(const std::string &deviceID, const std::string &packageName)
   return 0;
 }
 
-uint32_t StartAndroidPackageForCapture(const char *host, const char *package,
-                                       const CaptureOptions &opts)
+ExecuteResult StartAndroidPackageForCapture(const char *host, const char *package,
+                                            const CaptureOptions &opts)
 {
   int index = 0;
   std::string deviceID;
   Android::ExtractDeviceIDAndIndex(host, index, deviceID);
 
+  ExecuteResult ret;
+  ret.status = ReplayStatus::UnknownError;
+  ret.ident = RenderDoc_FirstTargetControlPort + RenderDoc_AndroidPortOffset * (index + 1);
+
   string packageName = basename(string(package));    // Remove leading '/' if any
 
   // adb shell cmd package resolve-activity -c android.intent.category.LAUNCHER com.jake.cube1
@@ -211,21 +215,25 @@ uint32_t StartAndroidPackageForCapture(const char *host, const char *package,
     if(!injected)
     {
       RDCERR("Failed to inject using JDWP");
-      return 0;
+      ret.status = ReplayStatus::JDWPFailure;
+      ret.ident = 0;
+      return ret;
     }
   }
 
-  uint32_t ret = RenderDoc_FirstTargetControlPort + RenderDoc_AndroidPortOffset * (index + 1);
+  ret.status = ReplayStatus::InjectionFailed;
+
   uint32_t elapsed = 0,
            timeout = 1000 *
                      RDCMAX(5, atoi(RenderDoc::Inst().GetConfigSetting("MaxConnectTimeout").c_str()));
   while(elapsed < timeout)
   {
     // Check if the target app has started yet and we can connect to it.
-    ITargetControl *control = RENDERDOC_CreateTargetControl(host, ret, "testConnection", false);
+    ITargetControl *control = RENDERDOC_CreateTargetControl(host, ret.ident, "testConnection", false);
     if(control)
     {
       control->Shutdown();
+      ret.status = ReplayStatus::Succeeded;
       break;
     }
 

renderdoc/android/android.h

@@ -31,8 +31,8 @@
 namespace Android
 {
 bool IsHostADB(const char *hostname);
-uint32_t StartAndroidPackageForCapture(const char *host, const char *package,
-                                       const CaptureOptions &opts);
+ExecuteResult StartAndroidPackageForCapture(const char *host, const char *package,
+                                            const CaptureOptions &opts);
 void ResetCaptureSettings(const std::string &deviceID);
 void ExtractDeviceIDAndIndex(const std::string &hostname, int &index, std::string &deviceID);
 Process::ProcessResult adbExecCommand(const std::string &deviceID, const std::string &args,

renderdoc/api/replay/renderdoc_replay.h

@@ -463,13 +463,6 @@ inline const WindowingData CreateAndroidWindowingData(ANativeWindow *window)
   return ret;
 }
 
-DOCUMENT(R"(Internal structure used for initialising environment in a replay application.)");
-struct GlobalEnvironment
-{
-  DOCUMENT("The handle to the X display to use internally. If left ``NULL``, one will be opened.");
-  Display *xlibDisplay = NULL;
-};
-
 // declare metatype/reflection for ResourceId here as the struct itself is declared before including
 // all relevant headers above
 #if defined(RENDERDOC_QT_COMPAT)
@@ -488,6 +481,27 @@ DECLARE_REFLECTION_STRUCT(ResourceId);
 #include "shader_types.h"
 #include "vk_pipestate.h"
 
+DOCUMENT(R"(Internal structure used for initialising environment in a replay application.)");
+struct GlobalEnvironment
+{
+  DOCUMENT("The handle to the X display to use internally. If left ``NULL``, one will be opened.");
+  Display *xlibDisplay = NULL;
+};
+
+DOCUMENT("The result of executing or injecting into a program.")
+struct ExecuteResult
+{
+  DOCUMENT(
+      "The :class:`ReplayStatus` resulting from the operation, indicating success or failure.");
+  ReplayStatus status;
+  DOCUMENT(R"(The ident where the new application is listening for target control, or 0 if something
+went wrong.
+)");
+  uint32_t ident;
+};
+
+DECLARE_REFLECTION_STRUCT(ExecuteResult);
+
 // there's not a good way to document a callback, so for lack of a better place we declare these
 // here and document them immediately below. They can be linked to from anywhere by name.
 typedef std::function<bool()> RENDERDOC_KillCallback;
@@ -1384,13 +1398,14 @@ This happens on the remote system, so all paths are relative to the remote files
   platform specific way to generate arguments.
 :param list env: Any :class:`EnvironmentModification` that should be made when running the program.
 :param CaptureOptions opts: The capture options to use when injecting into the program.
-:return: The ident where the new application is listening for target control, or 0 if something went
-  wrong.
-:rtype: ``int``
+:return: The :class:`ExecuteResult` indicating both the status of the operation (success or failure)
+  and any reason for failure, or else the ident where the new application is listening for target
+  control if everything succeeded.
+:rtype: ExecuteResult
 )");
-  virtual uint32_t ExecuteAndInject(const char *app, const char *workingDir, const char *cmdLine,
-                                    const rdcarray<EnvironmentModification> &env,
-                                    const CaptureOptions &opts) = 0;
+  virtual ExecuteResult ExecuteAndInject(const char *app, const char *workingDir, const char *cmdLine,
+                                         const rdcarray<EnvironmentModification> &env,
+                                         const CaptureOptions &opts) = 0;
 
   DOCUMENT(R"(Take ownership over a capture file.
 
@@ -1965,11 +1980,12 @@ DOCUMENT(R"(Launch an application and inject into it to allow capturing.
 :param list env: Any :class:`EnvironmentModification` that should be made when running the program.
 :param CaptureOptions opts: The capture options to use when injecting into the program.
 :param bool waitForExit: If ``True`` this function will block until the process exits.
-:return: The ident where the new application is listening for target control, or 0 if something went
-  wrong.
-:rtype: ``int``
+:return: The :class:`ExecuteResult` indicating both the status of the operation (success or failure)
+  and any reason for failure, or else the ident where the new application is listening for target
+  control if everything succeeded.
+:rtype: ExecuteResult
 )");
-extern "C" RENDERDOC_API uint32_t RENDERDOC_CC
+extern "C" RENDERDOC_API ExecuteResult RENDERDOC_CC
 RENDERDOC_ExecuteAndInject(const char *app, const char *workingDir, const char *cmdLine,
                            const rdcarray<EnvironmentModification> &env, const char *logfile,
                            const CaptureOptions &opts, bool waitForExit);
@@ -1980,11 +1996,12 @@ DOCUMENT(R"(Where supported by operating system and permissions, inject into a r
 :param list env: Any :class:`EnvironmentModification` that should be made when running the program.
 :param CaptureOptions opts: The capture options to use when injecting into the program.
 :param bool waitForExit: If ``True`` this function will block until the process exits.
-:return: The ident where the new application is listening for target control, or 0 if something went
-  wrong.
-:rtype: ``int``
+:return: The :class:`ExecuteResult` indicating both the status of the operation (success or failure)
+  and any reason for failure, or else the ident where the new application is listening for target
+  control if everything succeeded.
+:rtype: ExecuteResult
 )");
-extern "C" RENDERDOC_API uint32_t RENDERDOC_CC
+extern "C" RENDERDOC_API ExecuteResult RENDERDOC_CC
 RENDERDOC_InjectIntoProcess(uint32_t pid, const rdcarray<EnvironmentModification> &env,
                             const char *logfile, const CaptureOptions &opts, bool waitForExit);
 

renderdoc/api/replay/renderdoc_tostr.inl

@@ -49,6 +49,7 @@ std::string DoStringise(const ReplayStatus &el)
     STRINGISE_ENUM_CLASS_NAMED(APIDataCorrupted,
                                "Replaying the capture encountered invalid/corrupted data");
     STRINGISE_ENUM_CLASS_NAMED(APIReplayFailed, "Replaying the capture failed at the API level");
+    STRINGISE_ENUM_CLASS_NAMED(JDWPFailure, "JDWP debugger connection could not be established");
   }
   END_ENUM_STRINGISE();
 }

renderdoc/api/replay/replay_enums.h

@@ -2899,6 +2899,7 @@ enum class ReplayStatus : uint32_t
   APIHardwareUnsupported,
   APIDataCorrupted,
   APIReplayFailed,
+  JDWPFailure,
 };
 
 DECLARE_REFLECTION_ENUM(ReplayStatus);

renderdoc/core/remote_server.cpp

@@ -747,12 +747,12 @@ static void ActiveRemoteClientThread(ClientThread *threadData,
 
       reader.EndChunk();
 
-      uint32_t ident = 0;
+      ExecuteResult ret = {};
 
       if(threadData->allowExecution)
       {
-        ident = Process::LaunchAndInjectIntoProcess(app.c_str(), workingDir.c_str(),
-                                                    cmdLine.c_str(), env, "", opts, false);
+        ret = Process::LaunchAndInjectIntoProcess(app.c_str(), workingDir.c_str(), cmdLine.c_str(),
+                                                  env, "", opts, false);
       }
       else
       {
@@ -762,7 +762,7 @@ static void ActiveRemoteClientThread(ClientThread *threadData,
       {
         WRITE_DATA_SCOPE();
         SCOPED_SERIALISE_CHUNK(eRemoteServer_ExecuteAndInject);
-        SERIALISE_ELEMENT(ident);
+        SERIALISE_ELEMENT(ret);
       }
     }
     else if((int)type >= eReplayProxy_First && proxy)
@@ -1230,8 +1230,9 @@ public:
     return files;
   }
 
-  uint32_t ExecuteAndInject(const char *a, const char *w, const char *c,
-                            const rdcarray<EnvironmentModification> &env, const CaptureOptions &opts)
+  ExecuteResult ExecuteAndInject(const char *a, const char *w, const char *c,
+                                 const rdcarray<EnvironmentModification> &env,
+                                 const CaptureOptions &opts)
   {
     std::string app = a && a[0] ? a : "";
     std::string workingDir = w && w[0] ? w : "";
@@ -1249,7 +1250,7 @@ public:
           ok = Ping();
       });
 
-      uint32_t ret = Android::StartAndroidPackageForCapture(host, app.c_str(), opts);
+      ExecuteResult ret = Android::StartAndroidPackageForCapture(host, app.c_str(), opts);
 
       Atomic::Inc32(&done);
 
@@ -1269,7 +1270,7 @@ public:
       SERIALISE_ELEMENT(env);
     }
 
-    uint32_t ident = 0;
+    ExecuteResult ret = {};
 
     {
       READ_DATA_SCOPE();
@@ -1277,7 +1278,7 @@ public:
 
       if(type == eRemoteServer_ExecuteAndInject)
       {
-        SERIALISE_ELEMENT(ident);
+        SERIALISE_ELEMENT(ret);
       }
       else
       {
@@ -1287,7 +1288,7 @@ public:
       ser.EndChunk();
     }
 
-    return ident;
+    return ret;
   }
 
   void CopyCaptureFromRemote(const char *remotepath, const char *localpath,

renderdoc/os/os_specific.h

@@ -60,8 +60,8 @@ bool StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOp
 bool IsGlobalHookActive();
 void StopGlobalHook();
 
-uint32_t InjectIntoProcess(uint32_t pid, const rdcarray<EnvironmentModification> &env,
-                           const char *logfile, const CaptureOptions &opts, bool waitForExit);
+ExecuteResult InjectIntoProcess(uint32_t pid, const rdcarray<EnvironmentModification> &env,
+                                const char *logfile, const CaptureOptions &opts, bool waitForExit);
 struct ProcessResult
 {
   string strStdout, strStderror;
@@ -71,9 +71,10 @@ uint32_t LaunchProcess(const char *app, const char *workingDir, const char *cmdL
                        ProcessResult *result = NULL);
 uint32_t LaunchScript(const char *script, const char *workingDir, const char *args, bool internal,
                       ProcessResult *result = NULL);
-uint32_t LaunchAndInjectIntoProcess(const char *app, const char *workingDir, const char *cmdLine,
-                                    const rdcarray<EnvironmentModification> &env, const char *logfile,
-                                    const CaptureOptions &opts, bool waitForExit);
+ExecuteResult LaunchAndInjectIntoProcess(const char *app, const char *workingDir, const char *cmdLine,
+                                         const rdcarray<EnvironmentModification> &env,
+                                         const char *logfile, const CaptureOptions &opts,
+                                         bool waitForExit);
 void *LoadModule(const char *module);
 void *GetFunctionAddress(void *module, const char *function);
 uint32_t GetCurrentPID();

renderdoc/os/posix/posix_process.cpp

@@ -404,11 +404,12 @@ static pid_t RunProcess(const char *app, const char *workingDir, const char *cmd
   return childPid;
 }
 
-uint32_t Process::InjectIntoProcess(uint32_t pid, const rdcarray<EnvironmentModification> &env,
-                                    const char *logfile, const CaptureOptions &opts, bool waitForExit)
+ExecuteResult Process::InjectIntoProcess(uint32_t pid, const rdcarray<EnvironmentModification> &env,
+                                         const char *logfile, const CaptureOptions &opts,
+                                         bool waitForExit)
 {
   RDCUNIMPLEMENTED("Injecting into already running processes on linux");
-  return 0;
+  return {ReplayStatus::InjectionFailed, 0};
 }
 
 uint32_t Process::LaunchProcess(const char *app, const char *workingDir, const char *cmdLine,
@@ -472,16 +473,16 @@ uint32_t Process::LaunchScript(const char *script, const char *workingDir, const
   return LaunchProcess("bash", workingDir, args.c_str(), internal, result);
 }
 
-uint32_t Process::LaunchAndInjectIntoProcess(const char *app, const char *workingDir,
-                                             const char *cmdLine,
-                                             const rdcarray<EnvironmentModification> &envList,
-                                             const char *logfile, const CaptureOptions &opts,
-                                             bool waitForExit)
+ExecuteResult Process::LaunchAndInjectIntoProcess(const char *app, const char *workingDir,
+                                                  const char *cmdLine,
+                                                  const rdcarray<EnvironmentModification> &envList,
+                                                  const char *logfile, const CaptureOptions &opts,
+                                                  bool waitForExit)
 {
   if(app == NULL || app[0] == 0)
   {
     RDCERR("Invalid empty 'app'");
-    return 0;
+    return {ReplayStatus::InternalError, 0};
   }
 
   // turn environment string to a UTF-8 map
@@ -596,7 +597,7 @@ uint32_t Process::LaunchAndInjectIntoProcess(const char *app, const char *workin
   }
 
   CleanupStringArray(envp, NULL);
-  return ret;
+  return {ret == 0 ? ReplayStatus::InjectionFailed : ReplayStatus::Succeeded, (uint32_t)ret};
 }
 
 bool Process::StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions &opts)

renderdoc/os/win32/sys_win32_hooks.cpp

@@ -250,11 +250,12 @@ private:
       rdcarray<EnvironmentModification> env;
 
       // inherit logfile and capture options
-      uint32_t ident = RENDERDOC_InjectIntoProcess(lpProcessInformation->dwProcessId, env,
-                                                   RenderDoc::Inst().GetLogFile(),
-                                                   RenderDoc::Inst().GetCaptureOptions(), false);
+      ExecuteResult res = RENDERDOC_InjectIntoProcess(lpProcessInformation->dwProcessId, env,
+                                                      RenderDoc::Inst().GetLogFile(),
+                                                      RenderDoc::Inst().GetCaptureOptions(), false);
 
-      RenderDoc::Inst().AddChildProcess((uint32_t)lpProcessInformation->dwProcessId, ident);
+      if(res.status == ReplayStatus::Succeeded)
+        RenderDoc::Inst().AddChildProcess((uint32_t)lpProcessInformation->dwProcessId, res.ident);
     }
 
     if(resume)

renderdoc/os/win32/win32_process.cpp

@@ -500,8 +500,9 @@ static PROCESS_INFORMATION RunProcess(const char *app, const char *workingDir, c
   return pi;
 }
 
-uint32_t Process::InjectIntoProcess(uint32_t pid, const rdcarray<EnvironmentModification> &env,
-                                    const char *logfile, const CaptureOptions &opts, bool waitForExit)
+ExecuteResult Process::InjectIntoProcess(uint32_t pid, const rdcarray<EnvironmentModification> &env,
+                                         const char *logfile, const CaptureOptions &opts,
+                                         bool waitForExit)
 {
   wstring wlogfile = logfile == NULL ? L"" : StringFormat::UTF82Wide(logfile);
 
@@ -561,7 +562,7 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, const rdcarray<EnvironmentModi
     DWORD err = GetLastError();
     RDCERR("Couldn't determine bitness of process, err: %08x", err);
     CloseHandle(hProcess);
-    return 0;
+    return {ReplayStatus::IncompatibleProcess, 0};
   }
 
   bool capalt = false;
@@ -580,7 +581,7 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, const rdcarray<EnvironmentModi
   {
     DWORD err = GetLastError();
     RDCERR("Couldn't determine bitness of self, err: %08x", err);
-    return 0;
+    return {ReplayStatus::InternalError, 0};
   }
 
   // we know we're 32-bit, so if the target process is not wow64
@@ -632,7 +633,7 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, const rdcarray<EnvironmentModi
       RDCERR("Can't capture x64 process with x86 renderdoc");
 
       CloseHandle(hProcess);
-      return 0;
+      return {ReplayStatus::IncompatibleProcess, 0};
     }
   }
 #else
@@ -832,7 +833,7 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, const rdcarray<EnvironmentModi
       RDCERR(
           "Can't spawn alternate bitness renderdoccmd - have you built 32-bit and 64-bit?\n"
           "You need to build the matching bitness for the programs you want to capture.");
-      return 0;
+      return {ReplayStatus::InternalError, 0};
     }
 
     ResumeThread(pi.hThread);
@@ -848,7 +849,12 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, const rdcarray<EnvironmentModi
 
     CloseHandle(hProcess);
 
-    return (uint32_t)exitCode;
+    if(exitCode == 0)
+      return {ReplayStatus::UnknownError, 0};
+    if(exitCode < RenderDoc_FirstTargetControlPort)
+      return {(ReplayStatus)exitCode, 0};
+
+    return {ReplayStatus::Succeeded, (uint32_t)exitCode};
   }
 
   InjectDLL(hProcess, renderdocPath);
@@ -910,7 +916,7 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, const rdcarray<EnvironmentModi
 
   CloseHandle(hProcess);
 
-  return controlident;
+  return {ReplayStatus::Succeeded, controlident};
 }
 
 uint32_t Process::LaunchProcess(const char *app, const char *workingDir, const char *cmdLine,
@@ -984,11 +990,11 @@ uint32_t Process::LaunchScript(const char *script, const char *workingDir, const
   return LaunchProcess("cmd.exe", workingDir, args.c_str(), internal, result);
 }
 
-uint32_t Process::LaunchAndInjectIntoProcess(const char *app, const char *workingDir,
-                                             const char *cmdLine,
-                                             const rdcarray<EnvironmentModification> &env,
-                                             const char *logfile, const CaptureOptions &opts,
-                                             bool waitForExit)
+ExecuteResult Process::LaunchAndInjectIntoProcess(const char *app, const char *workingDir,
+                                                  const char *cmdLine,
+                                                  const rdcarray<EnvironmentModification> &env,
+                                                  const char *logfile, const CaptureOptions &opts,
+                                                  bool waitForExit)
 {
   void *func =
       GetProcAddress(GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll"), "INTERNAL_SetLogFile");
@@ -997,24 +1003,24 @@ uint32_t Process::LaunchAndInjectIntoProcess(const char *app, const char *workin
   {
     RDCERR("Can't find required export function in " STRINGIZE(
         RDOC_DLL_FILE) ".dll - corrupted/missing file?");
-    return 0;
+    return {ReplayStatus::InternalError, 0};
   }
 
   PROCESS_INFORMATION pi = RunProcess(app, workingDir, cmdLine, false, NULL, NULL);
 
   if(pi.dwProcessId == 0)
-    return 0;
+    return {ReplayStatus::InjectionFailed, 0};
 
-  uint32_t ret = InjectIntoProcess(pi.dwProcessId, env, logfile, opts, false);
+  ExecuteResult ret = InjectIntoProcess(pi.dwProcessId, env, logfile, opts, false);
 
   CloseHandle(pi.hProcess);
   ResumeThread(pi.hThread);
   ResumeThread(pi.hThread);
 
-  if(ret == 0)
+  if(ret.ident == 0 || ret.status != ReplayStatus::Succeeded)
   {
     CloseHandle(pi.hThread);
-    return 0;
+    return ret;
   }
 
   if(waitForExit)

renderdoc/replay/entry_points.cpp

@@ -296,7 +296,7 @@ extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_UnregisterMemoryRegion(void
     handler->UnregisterMemoryRegion(base);
 }
 
-extern "C" RENDERDOC_API uint32_t RENDERDOC_CC
+extern "C" RENDERDOC_API ExecuteResult RENDERDOC_CC
 RENDERDOC_ExecuteAndInject(const char *app, const char *workingDir, const char *cmdLine,
                            const rdcarray<EnvironmentModification> &env, const char *logfile,
                            const CaptureOptions &opts, bool waitForExit)
@@ -332,7 +332,7 @@ extern "C" RENDERDOC_API bool RENDERDOC_CC RENDERDOC_CanGlobalHook()
   return Process::CanGlobalHook();
 }
 
-extern "C" RENDERDOC_API uint32_t RENDERDOC_CC
+extern "C" RENDERDOC_API ExecuteResult RENDERDOC_CC
 RENDERDOC_InjectIntoProcess(uint32_t pid, const rdcarray<EnvironmentModification> &env,
                             const char *logfile, const CaptureOptions &opts, bool waitForExit)
 {

renderdoc/replay/renderdoc_serialise.inl

@@ -53,6 +53,15 @@ class undersized
 #define SIZE_CHECK(expected)
 #endif
 
+template <class SerialiserType>
+void DoSerialise(SerialiserType &ser, ExecuteResult &el)
+{
+  SERIALISE_MEMBER(status);
+  SERIALISE_MEMBER(ident);
+
+  SIZE_CHECK(8);
+}
+
 template <class SerialiserType>
 void DoSerialise(SerialiserType &ser, PathEntry &el)
 {
@@ -2057,6 +2066,7 @@ void DoSerialise(SerialiserType &ser, VKPipe::State &el)
 
 #pragma endregion Vulkan pipeline state
 
+INSTANTIATE_SERIALISE_TYPE(ExecuteResult)
 INSTANTIATE_SERIALISE_TYPE(PathEntry)
 INSTANTIATE_SERIALISE_TYPE(SectionProperties)
 INSTANTIATE_SERIALISE_TYPE(EnvironmentModification)

renderdoccmd/renderdoccmd.cpp

@@ -363,28 +363,28 @@ struct CaptureCommand : public Command
 
     rdcarray<EnvironmentModification> env;
 
-    uint32_t ident = RENDERDOC_ExecuteAndInject(
+    ExecuteResult res = RENDERDOC_ExecuteAndInject(
         executable.c_str(), workingDir.empty() ? "" : workingDir.c_str(),
         cmdLine.empty() ? "" : cmdLine.c_str(), env, logFile.empty() ? "" : logFile.c_str(), opts,
         parser.exist("wait-for-exit"));
 
-    if(ident == 0)
+    if(res.status != ReplayStatus::Succeeded)
     {
-      std::cerr << "Failed to create & inject." << std::endl;
-      return 2;
+      std::cerr << "Failed to create & inject: " << ToStr(res.status) << std::endl;
+      return (int)res.status;
     }
 
     if(parser.exist("wait-for-exit"))
     {
       std::cerr << "'" << executable << "' finished executing." << std::endl;
-      ident = 0;
+      res.ident = 0;
     }
     else
     {
-      std::cerr << "Launched as ID " << ident << std::endl;
+      std::cerr << "Launched as ID " << res.ident << std::endl;
     }
 
-    return ident;
+    return res.ident;
   }
 
   std::string EscapeArgument(const std::string &arg)
@@ -430,26 +430,26 @@ struct InjectCommand : public Command
 
     RENDERDOC_InitGlobalEnv(m_Env, convertArgs(parser.rest()));
 
-    uint32_t ident = RENDERDOC_InjectIntoProcess(PID, env, logFile.empty() ? "" : logFile.c_str(),
-                                                 opts, parser.exist("wait-for-exit"));
+    ExecuteResult res = RENDERDOC_InjectIntoProcess(
+        PID, env, logFile.empty() ? "" : logFile.c_str(), opts, parser.exist("wait-for-exit"));
 
-    if(ident == 0)
+    if(res.status != ReplayStatus::Succeeded)
     {
-      std::cerr << "Failed to inject." << std::endl;
-      return 2;
+      std::cerr << "Failed to inject: " << ToStr(res.status) << std::endl;
+      return (int)res.status;
     }
 
     if(parser.exist("wait-for-exit"))
     {
       std::cerr << PID << " finished executing." << std::endl;
-      ident = 0;
+      res.ident = 0;
     }
     else
     {
-      std::cerr << "Launched as ID " << ident << std::endl;
+      std::cerr << "Launched as ID " << res.ident << std::endl;
     }
 
-    return ident;
+    return res.ident;
   }
 };
 
@@ -879,10 +879,13 @@ struct CapAltBitCommand : public Command
 
     RENDERDOC_SetDebugLogFile(debuglog.c_str());
 
-    int ret = RENDERDOC_InjectIntoProcess(parser.get<uint32_t>("pid"), env,
-                                          parser.get<string>("log").c_str(), cmdopts, false);
+    ExecuteResult result = RENDERDOC_InjectIntoProcess(
+        parser.get<uint32_t>("pid"), env, parser.get<string>("log").c_str(), cmdopts, false);
 
-    return ret;
+    if(result.status == ReplayStatus::Succeeded)
+      return result.ident;
+
+    return (int)result.status;
   }
 };