From 20abaec49f45998203a2ad11b2c376faef6df1bc Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Thu, 12 Dec 2019 11:31:36 +0000
Subject: [PATCH] Don't load self-capturing module, only use if already present

---
 renderdoc/os/os_specific.h           | 1 +
 renderdoc/os/posix/posix_process.cpp | 5 +++++
 renderdoc/os/win32/win32_process.cpp | 5 +++++
 renderdoc/replay/entry_points.cpp    | 6 ++++++
 4 files changed, 17 insertions(+)

diff --git a/renderdoc/os/os_specific.h b/renderdoc/os/os_specific.h
index 76c659e10..1e7df27f7 100644
--- a/renderdoc/os/os_specific.h
+++ b/renderdoc/os/os_specific.h
@@ -74,6 +74,7 @@ ExecuteResult LaunchAndInjectIntoProcess(const char *app, const char *workingDir
                                          const rdcarray<EnvironmentModification> &env,
                                          const char *capturefile, const CaptureOptions &opts,
                                          bool waitForExit);
+bool IsModuleLoaded(const char *module);
 void *LoadModule(const char *module);
 void *GetFunctionAddress(void *module, const char *function);
 uint32_t GetCurrentPID();
diff --git a/renderdoc/os/posix/posix_process.cpp b/renderdoc/os/posix/posix_process.cpp
index d971a45bf..cf54506bb 100644
--- a/renderdoc/os/posix/posix_process.cpp
+++ b/renderdoc/os/posix/posix_process.cpp
@@ -855,6 +855,11 @@ void Process::StopGlobalHook()
 {
 }
 
+bool Process::IsModuleLoaded(const char *module)
+{
+  return dlopen(module, RTLD_NOW | RTLD_NOLOAD) != NULL;
+}
+
 void *Process::LoadModule(const char *module)
 {
   return dlopen(module, RTLD_NOW);
diff --git a/renderdoc/os/win32/win32_process.cpp b/renderdoc/os/win32/win32_process.cpp
index 774780464..29df2af8a 100644
--- a/renderdoc/os/win32/win32_process.cpp
+++ b/renderdoc/os/win32/win32_process.cpp
@@ -1656,6 +1656,11 @@ void Process::StopGlobalHook()
   globalHook = NULL;
 }
 
+bool Process::IsModuleLoaded(const char *module)
+{
+  return GetModuleHandleA(module) != NULL;
+}
+
 void *Process::LoadModule(const char *module)
 {
   HMODULE mod = GetModuleHandleA(module);
diff --git a/renderdoc/replay/entry_points.cpp b/renderdoc/replay/entry_points.cpp
index fde3a65e4..a0ecfc39e 100644
--- a/renderdoc/replay/entry_points.cpp
+++ b/renderdoc/replay/entry_points.cpp
@@ -486,6 +486,9 @@ RENDERDOC_BecomeRemoteServer(const char *listenhost, RENDERDOC_KillCallback kill
 
 extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StartSelfHostCapture(const char *dllname)
 {
+  if(!Process::IsModuleLoaded(dllname))
+    return;
+
   void *module = Process::LoadModule(dllname);
 
   if(module == NULL)
@@ -509,6 +512,9 @@ extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StartSelfHostCapture(const
 
 extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_EndSelfHostCapture(const char *dllname)
 {
+  if(!Process::IsModuleLoaded(dllname))
+    return;
+
   void *module = Process::LoadModule(dllname);
 
   if(module == NULL)