Change how replay programs are identified, from filename to sym export

* Instead of checking on the filename, we look for a specially named
  exported symbol somewhere in a module that's already loaded.
* This allows us to mark the python module as a replay program, so if
  it's loaded into the python interpreter it will be able to use the
  replay API.

qrenderdoc/Code/pyrenderdoc/pyrenderdoc_stub.cpp

@@ -24,6 +24,8 @@
 
 #include <Python.h>
 
+#include "renderdoc_replay.h"
+
 extern "C" PyThreadState *GetExecutingThreadState(PyObject *global_handle)
 {
   return NULL;
@@ -41,3 +43,5 @@ extern "C" bool IsThreadBlocking(PyObject *global_handle)
 extern "C" void SetThreadBlocking(PyObject *global_handle, bool block)
 {
 }
+
+REPLAY_PROGRAM_MARKER()

qrenderdoc/Code/qrenderdoc.cpp

@@ -34,6 +34,14 @@
 #include "Code/pyrenderdoc/PythonContext.h"
 #include "Windows/MainWindow.h"
 
+#if defined(Q_OS_WIN32)
+extern "C" {
+_declspec(dllexport) unsigned long NvOptimusEnablement = 0x00000001;
+}
+#endif
+
+REPLAY_PROGRAM_MARKER()
+
 void sharedLogOutput(QtMsgType type, const QMessageLogContext &context, const QString &msg)
 {
   LogType logtype = LogType::Comment;

renderdoc/api/replay/data_types.h

@@ -1021,6 +1021,8 @@ union PixelValue
   uint16_t value_u16[4];
 };
 
+DECLARE_REFLECTION_STRUCT(PixelValue);
+
 DOCUMENT("The value of pixel output at a particular event.");
 struct ModificationValue
 {

renderdoc/api/replay/renderdoc_replay.h

@@ -86,21 +86,15 @@ typedef uint8_t byte;
 
 #if defined(RENDERDOC_PLATFORM_WIN32)
 
-#ifdef RENDERDOC_EXPORTS
-#define RENDERDOC_API __declspec(dllexport)
-#else
-#define RENDERDOC_API __declspec(dllimport)
-#endif
+#define RENDERDOC_EXPORT_API __declspec(dllexport)
+#define RENDERDOC_IMPORT_API __declspec(dllimport)
 #define RENDERDOC_CC __cdecl
 
 #elif defined(RENDERDOC_PLATFORM_LINUX) || defined(RENDERDOC_PLATFORM_APPLE) || \
     defined(RENDERDOC_PLATFORM_ANDROID)
 
-#ifdef RENDERDOC_EXPORTS
-#define RENDERDOC_API __attribute__((visibility("default")))
-#else
-#define RENDERDOC_API
-#endif
+#define RENDERDOC_EXPORT_API __attribute__((visibility("default")))
+#define RENDERDOC_IMPORT_API
 
 #define RENDERDOC_CC
 
@@ -110,6 +104,18 @@ typedef uint8_t byte;
 
 #endif
 
+// this #define can be used to mark a program as a 'replay' program which should not be captured.
+// Any program used for such purpose must define and export this symbol in the main exe or one dll
+// that will be loaded before renderdoc.dll is loaded.
+#define REPLAY_PROGRAM_MARKER() \
+  extern "C" RENDERDOC_EXPORT_API void RENDERDOC_CC renderdoc__replay__marker() {}
+// define the API visibility depending on whether we're exporting
+#ifdef RENDERDOC_EXPORTS
+#define RENDERDOC_API RENDERDOC_EXPORT_API
+#else
+#define RENDERDOC_API RENDERDOC_IMPORT_API
+#endif
+
 // windowing structures
 
 #if defined(RENDERDOC_PLATFORM_WIN32)

renderdoc/hooks/hooks.h

@@ -58,6 +58,7 @@ private:
 #define HOOKS_BEGIN() Win32_IAT_BeginHooks()
 #define HOOKS_END() Win32_IAT_EndHooks()
 #define HOOKS_REMOVE() Win32_IAT_RemoveHooks()
+#define HOOKS_IDENTIFY(identifier) Win32_HookDetect(identifier)
 
 #elif ENABLED(RDOC_POSIX)
 
@@ -69,6 +70,7 @@ private:
 #define HOOKS_BEGIN() PosixHookInit()
 #define HOOKS_END()
 #define HOOKS_REMOVE()
+#define HOOKS_IDENTIFY(identifier) PosixHookDetect(identifier)
 
 #else
 

renderdoc/os/posix/android/android_hook.cpp

@@ -30,6 +30,11 @@ void PosixHookInit()
 {
 }
 
+bool PosixHookDetect(const char *identifier)
+{
+  return dlsym(RTLD_DEFAULT, identifier) != NULL;
+}
+
 void PosixHookLibrary(const char *name, dlopenCallback cb)
 {
   cb(dlopen(name, RTLD_NOW));

renderdoc/os/posix/apple/apple_hook.cpp

@@ -24,10 +24,17 @@
 
 #include "os/posix/posix_hook.h"
 
+#include <dlfcn.h>
+
 void PosixHookInit()
 {
 }
 
+bool PosixHookDetect(const char *identifier)
+{
+  return dlsym(RTLD_DEFAULT, identifier) != NULL;
+}
+
 void PosixHookLibrary(const char *name, dlopenCallback cb)
 {
 }

renderdoc/os/posix/linux/linux_hook.cpp

@@ -42,6 +42,11 @@ void PosixHookInit()
   hookInited = HOOK_MAGIC_NUMBER;
 }
 
+bool PosixHookDetect(const char *identifier)
+{
+  return dlsym(RTLD_DEFAULT, identifier) != NULL;
+}
+
 // need to lock around use of realdlopen and libraryHooks
 Threading::CriticalSection libLock;
 

renderdoc/os/posix/posix_hook.h

@@ -31,3 +31,5 @@ void PosixHookInit();
 // if this name is dlopen'd, the real library will be passed
 // to the callback and librenderdoc.so will be returned to user code
 void PosixHookLibrary(const char *name, dlopenCallback cb);
+
+bool PosixHookDetect(const char *identifier);

renderdoc/os/posix/posix_libentry.cpp

@@ -42,8 +42,7 @@ void library_loaded()
   string curfile;
   FileIO::GetExecutableFilename(curfile);
 
-  if(curfile.find("/renderdoccmd") != string::npos || curfile.find("/qrenderdoc") != string::npos ||
-     curfile.find("org.renderdoc.renderdoccmd") != string::npos)
+  if(HOOKS_IDENTIFY("renderdoc__replay__marker"))
   {
     RDCDEBUG("Not creating hooks - in replay app");
 

renderdoc/os/win32/win32_hook.cpp

@@ -28,6 +28,7 @@
 
 #include <tlhelp32.h>
 #include <algorithm>
+#include <functional>
 #include <map>
 #include <vector>
 #include "common/threading.h"
@@ -484,7 +485,13 @@ struct CachedHookData
 
 static CachedHookData *s_HookData = NULL;
 
-static void HookAllModules()
+#ifdef UNICODE
+#undef MODULEENTRY32
+#undef Module32First
+#undef Module32Next
+#endif
+
+static void ForAllModules(std::function<void(const MODULEENTRY32 &me32)> callback)
 {
   HANDLE hModuleSnap = INVALID_HANDLE_VALUE;
 
@@ -514,12 +521,6 @@ static void HookAllModules()
     return;
   }
 
-#ifdef UNICODE
-#undef MODULEENTRY32
-#undef Module32First
-#undef Module32Next
-#endif
-
   MODULEENTRY32 me32;
   RDCEraseEl(me32);
   me32.dwSize = sizeof(MODULEENTRY32);
@@ -539,12 +540,18 @@ static void HookAllModules()
 
   do
   {
-    s_HookData->ApplyHooks(me32.szModule, me32.hModule);
+    callback(me32);
   } while(ret == 0 && Module32Next(hModuleSnap, &me32));
 
   CloseHandle(hModuleSnap);
 }
 
+static void HookAllModules()
+{
+  ForAllModules(
+      [](const MODULEENTRY32 &me32) { s_HookData->ApplyHooks(me32.szModule, me32.hModule); });
+}
+
 HMODULE WINAPI Hooked_LoadLibraryExA(LPCSTR lpLibFileName, HANDLE fileHandle, DWORD flags)
 {
   bool dohook = true;
@@ -825,3 +832,13 @@ bool Win32_IAT_Hook(void **orig_function_ptr, const char *module_name, const cha
       FunctionHook(function, orig_function_ptr, destination_function_ptr));
   return true;
 }
+
+bool Win32_HookDetect(const char *identifier)
+{
+  bool ret = false;
+  ForAllModules([&ret, identifier](const MODULEENTRY32 &me32) {
+    if(GetProcAddress(me32.hModule, identifier) != NULL)
+      ret = true;
+  });
+  return ret;
+}

renderdoc/os/win32/win32_hook.h

@@ -30,3 +30,5 @@ bool Win32_IAT_Hook(void **orig_function_ptr, const char *module_name, const cha
 void Win32_IAT_BeginHooks();
 void Win32_IAT_EndHooks();
 void Win32_IAT_RemoveHooks();
+
+bool Win32_HookDetect(const char *identifier);

renderdoc/os/win32/win32_libentry.cpp

@@ -48,10 +48,8 @@ static BOOL add_hooks()
     return TRUE;
   }
 
-  if(f.find(CONCAT(L, STRINGIZE(RDOC_DLL_FILE)) L"cmd.exe") != wstring::npos ||
-     f.find(CONCAT(L, STRINGIZE(RDOC_DLL_FILE)) L"ui.vshost.exe") != wstring::npos ||
-     f.find(L"q" CONCAT(L, STRINGIZE(RDOC_DLL_FILE)) L".exe") != wstring::npos ||
-     f.find(CONCAT(L, STRINGIZE(RDOC_DLL_FILE)) L"ui.exe") != wstring::npos)
+  // search for an exported symbol with this name, typically renderdoc__replay__marker
+  if(HOOKS_IDENTIFY(STRINGIZE(RDOC_DLL_FILE) "__replay__marker"))
   {
     RDCDEBUG("Not creating hooks - in replay app");
 

renderdoccmd/renderdoccmd.cpp

@@ -729,6 +729,8 @@ struct CapAltBitCommand : public Command
   }
 };
 
+REPLAY_PROGRAM_MARKER()
+
 int renderdoccmd(const GlobalEnvironment &env, std::vector<std::string> &argv)
 {
   try
@@ -898,4 +900,4 @@ int renderdoccmd(const GlobalEnvironment &env, int argc, char **c_argv)
     argv[i] = c_argv[i];
 
   return renderdoccmd(env, argv);
-}
+}