diff --git a/qrenderdoc/Code/pyrenderdoc/pyrenderdoc_stub.cpp b/qrenderdoc/Code/pyrenderdoc/pyrenderdoc_stub.cpp index 10af8b92b..d864506d9 100644 --- a/qrenderdoc/Code/pyrenderdoc/pyrenderdoc_stub.cpp +++ b/qrenderdoc/Code/pyrenderdoc/pyrenderdoc_stub.cpp @@ -24,6 +24,8 @@ #include +#include "renderdoc_replay.h" + extern "C" PyThreadState *GetExecutingThreadState(PyObject *global_handle) { return NULL; @@ -41,3 +43,5 @@ extern "C" bool IsThreadBlocking(PyObject *global_handle) extern "C" void SetThreadBlocking(PyObject *global_handle, bool block) { } + +REPLAY_PROGRAM_MARKER() \ No newline at end of file diff --git a/qrenderdoc/Code/qrenderdoc.cpp b/qrenderdoc/Code/qrenderdoc.cpp index c2f234534..9fe80cc57 100644 --- a/qrenderdoc/Code/qrenderdoc.cpp +++ b/qrenderdoc/Code/qrenderdoc.cpp @@ -34,6 +34,14 @@ #include "Code/pyrenderdoc/PythonContext.h" #include "Windows/MainWindow.h" +#if defined(Q_OS_WIN32) +extern "C" { +_declspec(dllexport) unsigned long NvOptimusEnablement = 0x00000001; +} +#endif + +REPLAY_PROGRAM_MARKER() + void sharedLogOutput(QtMsgType type, const QMessageLogContext &context, const QString &msg) { LogType logtype = LogType::Comment; diff --git a/renderdoc/api/replay/data_types.h b/renderdoc/api/replay/data_types.h index c9137d4fc..16a31105e 100644 --- a/renderdoc/api/replay/data_types.h +++ b/renderdoc/api/replay/data_types.h @@ -1021,6 +1021,8 @@ union PixelValue uint16_t value_u16[4]; }; +DECLARE_REFLECTION_STRUCT(PixelValue); + DOCUMENT("The value of pixel output at a particular event."); struct ModificationValue { diff --git a/renderdoc/api/replay/renderdoc_replay.h b/renderdoc/api/replay/renderdoc_replay.h index 2809dca6c..70f659734 100644 --- a/renderdoc/api/replay/renderdoc_replay.h +++ b/renderdoc/api/replay/renderdoc_replay.h @@ -86,21 +86,15 @@ typedef uint8_t byte; #if defined(RENDERDOC_PLATFORM_WIN32) -#ifdef RENDERDOC_EXPORTS -#define RENDERDOC_API __declspec(dllexport) -#else -#define RENDERDOC_API __declspec(dllimport) -#endif +#define RENDERDOC_EXPORT_API __declspec(dllexport) +#define RENDERDOC_IMPORT_API __declspec(dllimport) #define RENDERDOC_CC __cdecl #elif defined(RENDERDOC_PLATFORM_LINUX) || defined(RENDERDOC_PLATFORM_APPLE) || \ defined(RENDERDOC_PLATFORM_ANDROID) -#ifdef RENDERDOC_EXPORTS -#define RENDERDOC_API __attribute__((visibility("default"))) -#else -#define RENDERDOC_API -#endif +#define RENDERDOC_EXPORT_API __attribute__((visibility("default"))) +#define RENDERDOC_IMPORT_API #define RENDERDOC_CC @@ -110,6 +104,18 @@ typedef uint8_t byte; #endif +// this #define can be used to mark a program as a 'replay' program which should not be captured. +// Any program used for such purpose must define and export this symbol in the main exe or one dll +// that will be loaded before renderdoc.dll is loaded. +#define REPLAY_PROGRAM_MARKER() \ + extern "C" RENDERDOC_EXPORT_API void RENDERDOC_CC renderdoc__replay__marker() {} +// define the API visibility depending on whether we're exporting +#ifdef RENDERDOC_EXPORTS +#define RENDERDOC_API RENDERDOC_EXPORT_API +#else +#define RENDERDOC_API RENDERDOC_IMPORT_API +#endif + // windowing structures #if defined(RENDERDOC_PLATFORM_WIN32) diff --git a/renderdoc/hooks/hooks.h b/renderdoc/hooks/hooks.h index 63006d138..f40f6839f 100644 --- a/renderdoc/hooks/hooks.h +++ b/renderdoc/hooks/hooks.h @@ -58,6 +58,7 @@ private: #define HOOKS_BEGIN() Win32_IAT_BeginHooks() #define HOOKS_END() Win32_IAT_EndHooks() #define HOOKS_REMOVE() Win32_IAT_RemoveHooks() +#define HOOKS_IDENTIFY(identifier) Win32_HookDetect(identifier) #elif ENABLED(RDOC_POSIX) @@ -69,6 +70,7 @@ private: #define HOOKS_BEGIN() PosixHookInit() #define HOOKS_END() #define HOOKS_REMOVE() +#define HOOKS_IDENTIFY(identifier) PosixHookDetect(identifier) #else diff --git a/renderdoc/os/posix/android/android_hook.cpp b/renderdoc/os/posix/android/android_hook.cpp index 86e12d048..6cddea7f0 100644 --- a/renderdoc/os/posix/android/android_hook.cpp +++ b/renderdoc/os/posix/android/android_hook.cpp @@ -30,6 +30,11 @@ void PosixHookInit() { } +bool PosixHookDetect(const char *identifier) +{ + return dlsym(RTLD_DEFAULT, identifier) != NULL; +} + void PosixHookLibrary(const char *name, dlopenCallback cb) { cb(dlopen(name, RTLD_NOW)); diff --git a/renderdoc/os/posix/apple/apple_hook.cpp b/renderdoc/os/posix/apple/apple_hook.cpp index 898c1a9a5..7f8c4fb40 100644 --- a/renderdoc/os/posix/apple/apple_hook.cpp +++ b/renderdoc/os/posix/apple/apple_hook.cpp @@ -24,10 +24,17 @@ #include "os/posix/posix_hook.h" +#include + void PosixHookInit() { } +bool PosixHookDetect(const char *identifier) +{ + return dlsym(RTLD_DEFAULT, identifier) != NULL; +} + void PosixHookLibrary(const char *name, dlopenCallback cb) { } \ No newline at end of file diff --git a/renderdoc/os/posix/linux/linux_hook.cpp b/renderdoc/os/posix/linux/linux_hook.cpp index 76b67b64c..2a73e484d 100644 --- a/renderdoc/os/posix/linux/linux_hook.cpp +++ b/renderdoc/os/posix/linux/linux_hook.cpp @@ -42,6 +42,11 @@ void PosixHookInit() hookInited = HOOK_MAGIC_NUMBER; } +bool PosixHookDetect(const char *identifier) +{ + return dlsym(RTLD_DEFAULT, identifier) != NULL; +} + // need to lock around use of realdlopen and libraryHooks Threading::CriticalSection libLock; diff --git a/renderdoc/os/posix/posix_hook.h b/renderdoc/os/posix/posix_hook.h index 9b761e1fb..90bb22667 100644 --- a/renderdoc/os/posix/posix_hook.h +++ b/renderdoc/os/posix/posix_hook.h @@ -31,3 +31,5 @@ void PosixHookInit(); // if this name is dlopen'd, the real library will be passed // to the callback and librenderdoc.so will be returned to user code void PosixHookLibrary(const char *name, dlopenCallback cb); + +bool PosixHookDetect(const char *identifier); \ No newline at end of file diff --git a/renderdoc/os/posix/posix_libentry.cpp b/renderdoc/os/posix/posix_libentry.cpp index 2264dd20e..93e9605ce 100644 --- a/renderdoc/os/posix/posix_libentry.cpp +++ b/renderdoc/os/posix/posix_libentry.cpp @@ -42,8 +42,7 @@ void library_loaded() string curfile; FileIO::GetExecutableFilename(curfile); - if(curfile.find("/renderdoccmd") != string::npos || curfile.find("/qrenderdoc") != string::npos || - curfile.find("org.renderdoc.renderdoccmd") != string::npos) + if(HOOKS_IDENTIFY("renderdoc__replay__marker")) { RDCDEBUG("Not creating hooks - in replay app"); diff --git a/renderdoc/os/win32/win32_hook.cpp b/renderdoc/os/win32/win32_hook.cpp index e3fc5d0d1..bec062006 100644 --- a/renderdoc/os/win32/win32_hook.cpp +++ b/renderdoc/os/win32/win32_hook.cpp @@ -28,6 +28,7 @@ #include #include +#include #include #include #include "common/threading.h" @@ -484,7 +485,13 @@ struct CachedHookData static CachedHookData *s_HookData = NULL; -static void HookAllModules() +#ifdef UNICODE +#undef MODULEENTRY32 +#undef Module32First +#undef Module32Next +#endif + +static void ForAllModules(std::function callback) { HANDLE hModuleSnap = INVALID_HANDLE_VALUE; @@ -514,12 +521,6 @@ static void HookAllModules() return; } -#ifdef UNICODE -#undef MODULEENTRY32 -#undef Module32First -#undef Module32Next -#endif - MODULEENTRY32 me32; RDCEraseEl(me32); me32.dwSize = sizeof(MODULEENTRY32); @@ -539,12 +540,18 @@ static void HookAllModules() do { - s_HookData->ApplyHooks(me32.szModule, me32.hModule); + callback(me32); } while(ret == 0 && Module32Next(hModuleSnap, &me32)); CloseHandle(hModuleSnap); } +static void HookAllModules() +{ + ForAllModules( + [](const MODULEENTRY32 &me32) { s_HookData->ApplyHooks(me32.szModule, me32.hModule); }); +} + HMODULE WINAPI Hooked_LoadLibraryExA(LPCSTR lpLibFileName, HANDLE fileHandle, DWORD flags) { bool dohook = true; @@ -825,3 +832,13 @@ bool Win32_IAT_Hook(void **orig_function_ptr, const char *module_name, const cha FunctionHook(function, orig_function_ptr, destination_function_ptr)); return true; } + +bool Win32_HookDetect(const char *identifier) +{ + bool ret = false; + ForAllModules([&ret, identifier](const MODULEENTRY32 &me32) { + if(GetProcAddress(me32.hModule, identifier) != NULL) + ret = true; + }); + return ret; +} \ No newline at end of file diff --git a/renderdoc/os/win32/win32_hook.h b/renderdoc/os/win32/win32_hook.h index a8ab3dd18..6bd6daab0 100644 --- a/renderdoc/os/win32/win32_hook.h +++ b/renderdoc/os/win32/win32_hook.h @@ -30,3 +30,5 @@ bool Win32_IAT_Hook(void **orig_function_ptr, const char *module_name, const cha void Win32_IAT_BeginHooks(); void Win32_IAT_EndHooks(); void Win32_IAT_RemoveHooks(); + +bool Win32_HookDetect(const char *identifier); \ No newline at end of file diff --git a/renderdoc/os/win32/win32_libentry.cpp b/renderdoc/os/win32/win32_libentry.cpp index 9d51afba2..a374666c6 100644 --- a/renderdoc/os/win32/win32_libentry.cpp +++ b/renderdoc/os/win32/win32_libentry.cpp @@ -48,10 +48,8 @@ static BOOL add_hooks() return TRUE; } - if(f.find(CONCAT(L, STRINGIZE(RDOC_DLL_FILE)) L"cmd.exe") != wstring::npos || - f.find(CONCAT(L, STRINGIZE(RDOC_DLL_FILE)) L"ui.vshost.exe") != wstring::npos || - f.find(L"q" CONCAT(L, STRINGIZE(RDOC_DLL_FILE)) L".exe") != wstring::npos || - f.find(CONCAT(L, STRINGIZE(RDOC_DLL_FILE)) L"ui.exe") != wstring::npos) + // search for an exported symbol with this name, typically renderdoc__replay__marker + if(HOOKS_IDENTIFY(STRINGIZE(RDOC_DLL_FILE) "__replay__marker")) { RDCDEBUG("Not creating hooks - in replay app"); diff --git a/renderdoccmd/renderdoccmd.cpp b/renderdoccmd/renderdoccmd.cpp index b141a42e7..f9ebff718 100644 --- a/renderdoccmd/renderdoccmd.cpp +++ b/renderdoccmd/renderdoccmd.cpp @@ -729,6 +729,8 @@ struct CapAltBitCommand : public Command } }; +REPLAY_PROGRAM_MARKER() + int renderdoccmd(const GlobalEnvironment &env, std::vector &argv) { try @@ -898,4 +900,4 @@ int renderdoccmd(const GlobalEnvironment &env, int argc, char **c_argv) argv[i] = c_argv[i]; return renderdoccmd(env, argv); -} +} \ No newline at end of file