Don't load self-capturing module, only use if already present

renderdoc/os/os_specific.h

@@ -74,6 +74,7 @@ ExecuteResult LaunchAndInjectIntoProcess(const char *app, const char *workingDir
                                          const rdcarray<EnvironmentModification> &env,
                                          const char *capturefile, const CaptureOptions &opts,
                                          bool waitForExit);
+bool IsModuleLoaded(const char *module);
 void *LoadModule(const char *module);
 void *GetFunctionAddress(void *module, const char *function);
 uint32_t GetCurrentPID();

renderdoc/os/posix/posix_process.cpp

@@ -855,6 +855,11 @@ void Process::StopGlobalHook()
 {
 }
 
+bool Process::IsModuleLoaded(const char *module)
+{
+  return dlopen(module, RTLD_NOW | RTLD_NOLOAD) != NULL;
+}
+
 void *Process::LoadModule(const char *module)
 {
   return dlopen(module, RTLD_NOW);

renderdoc/os/win32/win32_process.cpp

@@ -1656,6 +1656,11 @@ void Process::StopGlobalHook()
   globalHook = NULL;
 }
 
+bool Process::IsModuleLoaded(const char *module)
+{
+  return GetModuleHandleA(module) != NULL;
+}
+
 void *Process::LoadModule(const char *module)
 {
   HMODULE mod = GetModuleHandleA(module);

renderdoc/replay/entry_points.cpp

@@ -486,6 +486,9 @@ RENDERDOC_BecomeRemoteServer(const char *listenhost, RENDERDOC_KillCallback kill
 
 extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StartSelfHostCapture(const char *dllname)
 {
+  if(!Process::IsModuleLoaded(dllname))
+    return;
+
   void *module = Process::LoadModule(dllname);
 
   if(module == NULL)
@@ -509,6 +512,9 @@ extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StartSelfHostCapture(const
 
 extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_EndSelfHostCapture(const char *dllname)
 {
+  if(!Process::IsModuleLoaded(dllname))
+    return;
+
   void *module = Process::LoadModule(dllname);
 
   if(module == NULL)