diff --git a/renderdoc/api/replay/renderdoc_replay.h b/renderdoc/api/replay/renderdoc_replay.h index f20aa7522..a88725099 100644 --- a/renderdoc/api/replay/renderdoc_replay.h +++ b/renderdoc/api/replay/renderdoc_replay.h @@ -1324,17 +1324,49 @@ DOCUMENT(R"(Retrieve the default and recommended set of capture options. )"); extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_GetDefaultCaptureOptions(CaptureOptions *opts); -DOCUMENT(R"(Where supported by platform, configuration and setup, begin injecting speculatively into -all new processes started on the system. +DOCUMENT(R"(Begin injecting speculatively into all new processes started on the system. Where +supported by platform, configuration, and setup begin injecting speculatively into all new processes +started on the system. + +This function can only be called if global hooking is supported (see :ref:`CanGlobalHook`) and if +global hooking is not active (see :ref:`IsGlobalHookActive`). + +This function must be called when the process is running with administrator/superuser permissions. :param str pathmatch: A string to match against each new process's executable path to determine which corresponds to the program we actually want to capture. :param str logfile: Where to store any captures. :param CaptureOptions opts: The capture options to use when injecting into the program. +:return: ``True`` if the hook is active, ``False`` if something went wrong. The hook must be closed + with :ref:`StopGlobalHook` before the application is closed. +:rtype: ``bool`` )"); -extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StartGlobalHook(const char *pathmatch, - const char *logfile, - const CaptureOptions &opts); +extern "C" RENDERDOC_API bool32 RENDERDOC_CC RENDERDOC_StartGlobalHook(const char *pathmatch, + const char *logfile, + const CaptureOptions &opts); + +DOCUMENT(R"(Stop the global hook that was activated by :ref:`StartGlobalHook`. + +This function can only be called if global hooking is supported (see :ref:`CanGlobalHook`) and if +global hooking is active (see :ref:`IsGlobalHookActive`). +)"); +extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StopGlobalHook(); + +DOCUMENT(R"(Determines if the global hook is active or not. + +This function can only be called if global hooking is supported (see :ref:`CanGlobalHook`). + +:return: ``True`` if the hook is active, or ``False`` if the hook is inactive. +:rtype: ``bool`` +)"); +extern "C" RENDERDOC_API bool32 RENDERDOC_CC RENDERDOC_IsGlobalHookActive(); + +DOCUMENT(R"(Determines if the global hook is supported on the current platform and configuration. + +:return: ``True`` if global hooking can be used on the platform, ``False`` if not. +:rtype: ``bool`` +)"); +extern "C" RENDERDOC_API bool32 RENDERDOC_CC RENDERDOC_CanGlobalHook(); DOCUMENT(R"(Launch an application and inject into it to allow capturing. diff --git a/renderdoc/os/os_specific.h b/renderdoc/os/os_specific.h index 65fde797c..ff73a2c91 100644 --- a/renderdoc/os/os_specific.h +++ b/renderdoc/os/os_specific.h @@ -52,7 +52,11 @@ void RegisterEnvironmentModification(EnvironmentModification modif); void ApplyEnvironmentModification(); -void StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions &opts); +bool CanGlobalHook(); +bool StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions &opts); +bool IsGlobalHookActive(); +void StopGlobalHook(); + uint32_t InjectIntoProcess(uint32_t pid, const rdctype::array &env, const char *logfile, const CaptureOptions &opts, bool waitForExit); struct ProcessResult diff --git a/renderdoc/os/posix/posix_process.cpp b/renderdoc/os/posix/posix_process.cpp index 089c1c98f..1061200ed 100644 --- a/renderdoc/os/posix/posix_process.cpp +++ b/renderdoc/os/posix/posix_process.cpp @@ -601,9 +601,24 @@ uint32_t Process::LaunchAndInjectIntoProcess(const char *app, const char *workin return ret; } -void Process::StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions &opts) +bool Process::StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions &opts) { RDCUNIMPLEMENTED("Global hooking of all processes on linux"); + return false; +} + +bool Process::CanGlobalHook() +{ + return false; +} + +bool Process::IsGlobalHookActive() +{ + return false; +} + +void Process::StopGlobalHook() +{ } void *Process::LoadModule(const char *module) diff --git a/renderdoc/os/win32/win32_process.cpp b/renderdoc/os/win32/win32_process.cpp index 80142afef..a9fe9d0fc 100644 --- a/renderdoc/os/win32/win32_process.cpp +++ b/renderdoc/os/win32/win32_process.cpp @@ -996,10 +996,276 @@ uint32_t Process::LaunchAndInjectIntoProcess(const char *app, const char *workin return ret; } -void Process::StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions &opts) +bool Process::CanGlobalHook() +{ + // all we need is admin rights and it's the caller's responsibility to ensure that. + return true; +} + +// to simplify the below code, rather than splitting by 32-bit/64-bit we split by native and Wow32. +// This means that for 32-bit code (whether it's on 32-bit OS or not) we just have native, and the +// Wow32 stuff is empty/unused. For 64-bit we use both. Thus the native registry key is always the +// same path regardless of the bitness we're running as and we don't have to move things around or +// have conditionals all over + +struct GlobalHookData +{ + struct + { + HANDLE pipe = NULL; + DWORD appinitEnabled = 0; + wstring appinitDLLs; + } dataNative, dataWow32; + + volatile int32_t finished = 0; + Threading::ThreadHandle pipeThread = 0; +}; + +// utility function to close the registry keys, print an error, and quit +static bool HandleRegError(HKEY keyNative, HKEY keyWow32, LSTATUS ret, const char *msg) +{ + if(keyNative) + RegCloseKey(keyNative); + + if(keyWow32) + RegCloseKey(keyWow32); + + RDCERR("Error with AppInit registry keys - %s (%d)", msg, ret); + return false; +} + +#define REG_CHECK(msg) \ + if(ret != ERROR_SUCCESS) \ + { \ + return HandleRegError(keyNative, keyWow32, ret, msg); \ + } + +// function to backup the previous settings for AppInit, then enable it and write our own paths. +bool BackupAndChangeRegistry(GlobalHookData &hookdata, const wstring &shimpathWow32, + const wstring &shimpathNative) +{ + HKEY keyNative = NULL; + HKEY keyWow32 = NULL; + + // open the native key + LSTATUS ret = RegCreateKeyExA(HKEY_LOCAL_MACHINE, + "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows", 0, NULL, + 0, KEY_READ | KEY_WRITE, NULL, &keyNative, NULL); + + REG_CHECK("Could not open AppInit key"); + + // if we are doing Wow32, open that key as well + if(!shimpathWow32.empty()) + { + ret = RegCreateKeyExA(HKEY_LOCAL_MACHINE, + "SOFTWARE\\Wow6432Node\\Microsoft\\Windows NT\\CurrentVersion\\Windows", + 0, NULL, 0, KEY_READ | KEY_WRITE, NULL, &keyWow32, NULL); + + REG_CHECK("Could not open AppInit key"); + } + + const DWORD one = 1; + + // fetch the previous data for LoadAppInit_DLLs and AppInit_DLLs + DWORD sz = 4; + ret = RegGetValueA(keyNative, NULL, "LoadAppInit_DLLs", RRF_RT_REG_DWORD, NULL, + (void *)&hookdata.dataNative.appinitEnabled, &sz); + REG_CHECK("Could not fetch LoadAppInit_DLLs"); + + sz = 0; + ret = RegGetValueW(keyNative, NULL, L"AppInit_DLLs", RRF_RT_ANY, NULL, NULL, &sz); + if(ret == ERROR_MORE_DATA || ret == ERROR_SUCCESS) + { + hookdata.dataNative.appinitDLLs.resize(sz / sizeof(wchar_t)); + ret = RegGetValueW(keyNative, NULL, L"AppInit_DLLs", RRF_RT_ANY, NULL, + (void *)&hookdata.dataNative.appinitDLLs[0], &sz); + } + REG_CHECK("Could not fetch AppInit_DLLs"); + + // set DWORD:1 for LoadAppInit_DLLs and convert our path to a short path then set it + ret = RegSetValueExA(keyNative, "LoadAppInit_DLLs", 0, REG_DWORD, (const BYTE *)&one, sizeof(one)); + REG_CHECK("Could not set LoadAppInit_DLLs"); + + wstring shortpath; + shortpath = shimpathNative; + GetShortPathNameW(shimpathNative.c_str(), (wchar_t *)&shortpath[0], (DWORD)shortpath.size()); + + ret = RegSetValueExW(keyNative, L"AppInit_DLLs", 0, REG_SZ, (const BYTE *)shortpath.data(), + DWORD(shortpath.size() * sizeof(wchar_t))); + REG_CHECK("Could not set AppInit_DLLs"); + + // if we're doing Wow32, repeat the process for those keys + if(keyWow32) + { + sz = 4; + ret = RegGetValueA(keyWow32, NULL, "LoadAppInit_DLLs", RRF_RT_REG_DWORD, NULL, + (void *)&hookdata.dataWow32.appinitEnabled, &sz); + REG_CHECK("Could not fetch LoadAppInit_DLLs"); + + sz = 0; + ret = RegGetValueW(keyWow32, NULL, L"AppInit_DLLs", RRF_RT_ANY, NULL, NULL, &sz); + if(ret == ERROR_MORE_DATA || ret == ERROR_SUCCESS) + { + hookdata.dataWow32.appinitDLLs.resize(sz / sizeof(wchar_t)); + ret = RegGetValueW(keyWow32, NULL, L"AppInit_DLLs", RRF_RT_ANY, NULL, + (void *)&hookdata.dataWow32.appinitDLLs[0], &sz); + } + REG_CHECK("Could not fetch AppInit_DLLs"); + + ret = RegSetValueExA(keyWow32, "LoadAppInit_DLLs", 0, REG_DWORD, (const BYTE *)&one, sizeof(one)); + REG_CHECK("Could not set LoadAppInit_DLLs"); + + shortpath = shimpathWow32; + GetShortPathNameW(shimpathWow32.c_str(), &shortpath[0], (DWORD)shortpath.size()); + + ret = RegSetValueExW(keyWow32, L"AppInit_DLLs", 0, REG_SZ, (const BYTE *)shortpath.data(), + DWORD(shortpath.size() * sizeof(wchar_t))); + REG_CHECK("Could not set AppInit_DLLs"); + } + + wstring backup; + + // write a .reg file that contains the previous settings, so that if all else fails the user can + // manually insert it back into the registry to restore everything. + backup += L"Windows Registry Editor Version 5.00\n"; + backup += L"\n"; + backup += L"[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows]\n"; + backup += L"\"LoadAppInit_DLLs\"=dword:0000000"; + backup += (hookdata.dataNative.appinitEnabled ? L"1\n" : L"0\n"); + backup += L"\"AppInit_DLLs\"=\""; + // we append with the C string so we don't add trailing NULLs into the text. + backup += hookdata.dataNative.appinitDLLs.c_str(); + backup += L"\"\n"; + if(keyWow32) + { + backup += L"\n"; + backup += + L"[HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\Microsoft\\" + L"Windows NT\\CurrentVersion\\Windows]\n"; + backup += L"\"LoadAppInit_DLLs\"=dword:0000000"; + backup += (hookdata.dataWow32.appinitEnabled ? L"1\n" : L"0\n"); + backup += L"\"AppInit_DLLs\"=\""; + backup += hookdata.dataWow32.appinitDLLs.c_str(); + backup += L"\"\n"; + } + + if(keyNative) + RegCloseKey(keyNative); + + if(keyWow32) + RegCloseKey(keyWow32); + + keyNative = keyWow32 = NULL; + + // write it to disk but don't fail if we can't, just print it to the log and keep going. + wchar_t reg_backup[MAX_PATH]; + GetTempPathW(MAX_PATH, reg_backup); + wcscat_s(reg_backup, L"RenderDoc_RestoreGlobalHook.reg"); + + FILE *f = NULL; + _wfopen_s(&f, reg_backup, L"w"); + if(f) + { + fputws(backup.c_str(), f); + fclose(f); + } + else + { + RDCERR("Error opening registry backup file %ls", reg_backup); + RDCERR("Backup registry data is:\n\n%ls\n\n", backup.c_str()); + } + + return true; +} + +// switch error-handling to print-and-continue, as we can't really do anything about it at this +// point and we want to continue restoring in case only one thing failed. +#undef REG_CHECK +#define REG_CHECK(msg) \ + if(ret != ERROR_SUCCESS) \ + { \ + HandleRegError(keyNative, keyWow32, ret, "Could not open AppInit key"); \ + } + +void RestoreRegistry(const GlobalHookData &hookdata) +{ + HKEY keyNative = NULL; + HKEY keyWow32 = NULL; + LSTATUS ret = RegCreateKeyExA(HKEY_LOCAL_MACHINE, + "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows", 0, NULL, + 0, KEY_READ | KEY_WRITE, NULL, &keyNative, NULL); + + REG_CHECK("Could not open AppInit key"); + +#if ENABLED(RDOC_X64) + ret = RegCreateKeyExA(HKEY_LOCAL_MACHINE, + "SOFTWARE\\Wow6432Node\\Microsoft\\Windows NT\\CurrentVersion\\Windows", 0, + NULL, 0, KEY_READ | KEY_WRITE, NULL, &keyWow32, NULL); + + REG_CHECK("Could not open AppInit key"); +#endif + + // set the native values back to where they were + ret = RegSetValueExA(keyNative, "LoadAppInit_DLLs", 0, REG_DWORD, + (const BYTE *)&hookdata.dataNative.appinitEnabled, + sizeof(hookdata.dataNative.appinitEnabled)); + REG_CHECK("Could not set LoadAppInit_DLLs"); + + ret = RegSetValueExW(keyNative, L"AppInit_DLLs", 0, REG_SZ, + (const BYTE *)hookdata.dataNative.appinitDLLs.data(), + DWORD(hookdata.dataNative.appinitDLLs.size() * sizeof(wchar_t))); + REG_CHECK("Could not set AppInit_DLLs"); + + // if we opened it, restore the Wow32 values as well + if(keyWow32) + { + ret = RegSetValueExA(keyWow32, "LoadAppInit_DLLs", 0, REG_DWORD, + (const BYTE *)&hookdata.dataWow32.appinitEnabled, + sizeof(hookdata.dataWow32.appinitEnabled)); + REG_CHECK("Could not set LoadAppInit_DLLs"); + + ret = RegSetValueExW(keyWow32, L"AppInit_DLLs", 0, REG_SZ, + (const BYTE *)hookdata.dataWow32.appinitDLLs.data(), + DWORD(hookdata.dataWow32.appinitDLLs.size() * sizeof(wchar_t))); + REG_CHECK("Could not set AppInit_DLLs"); + } +} + +static GlobalHookData *globalHook = NULL; + +// a thread we run in the background just to keep the pipes open and wait until we're ready to stop +// the global hook. +static void GlobalHookThread(void *) +{ + // keep looping doing an atomic compare-exchange to check that finished is still 0 + while(Atomic::CmpExch32(&globalHook->finished, 0, 0) == 0) + { + // wake every quarter of a second to test again + Threading::Sleep(250); + } + + char exitData[32] = "exit"; + + // write some data into the pipe and close it. The data is (currently) unimportant, just that it + // causes the blocking read on the other end to succeed and close the program. + DWORD dummy = 0; + if(globalHook->dataNative.pipe) + { + WriteFile(globalHook->dataNative.pipe, exitData, (DWORD)sizeof(exitData), &dummy, NULL); + CloseHandle(globalHook->dataNative.pipe); + } + + if(globalHook->dataWow32.pipe) + { + WriteFile(globalHook->dataWow32.pipe, exitData, (DWORD)sizeof(exitData), &dummy, NULL); + CloseHandle(globalHook->dataWow32.pipe); + } +} + +bool Process::StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions &opts) { if(pathmatch == NULL) - return; + return false; wchar_t renderdocPath[MAX_PATH] = {0}; GetModuleFileNameW(GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll"), &renderdocPath[0], @@ -1012,7 +1278,82 @@ void Process::StartGlobalHook(const char *pathmatch, const char *logfile, const else slash = renderdocPath + wcslen(renderdocPath); - wcscat_s(renderdocPath, L"\\renderdoccmd.exe"); + // the native renderdoccmd.exe is always next to the dll. Wow32 will be somewhere else + wstring cmdpathNative = renderdocPath; + cmdpathNative += L"\\renderdoccmd.exe"; + wstring cmdpathWow32; + + wstring shimpathNative = renderdocPath; + wstring shimpathWow32; + +#if ENABLED(RDOC_X64) + + // native shim is just renderdocshim64.dll + *slash = 0; + wcscat_s(renderdocPath, L"\\renderdocshim64.dll"); + shimpathNative = renderdocPath; + + // if it looks like we're in the development environment, look for the alternate bitness in the + // corresponding folder + const wchar_t *devLocation = wcsstr(renderdocPath, L"\\x64\\Development\\"); + if(devLocation) + { + size_t idx = devLocation - renderdocPath; + + renderdocPath[idx] = 0; + + shimpathWow32 = renderdocPath; + shimpathWow32 += L"\\Win32\\Development\\renderdocshim32.dll"; + + cmdpathWow32 = renderdocPath; + cmdpathWow32 += L"\\Win32\\Development\\renderdoccmd.exe"; + } + + if(!devLocation) + { + devLocation = wcsstr(renderdocPath, L"\\x64\\Release\\"); + + if(devLocation) + { + size_t idx = devLocation - renderdocPath; + + renderdocPath[idx] = 0; + + shimpathWow32 = renderdocPath; + shimpathWow32 += L"\\Win32\\Release\\renderdocshim32.dll"; + + cmdpathWow32 = renderdocPath; + cmdpathWow32 += L"\\Win32\\Release\\renderdoccmd.exe"; + } + } + + // if we're not in the dev environment, assume it's under a x86\ subfolder + if(!devLocation) + { + *slash = 0; + shimpathWow32 = renderdocPath; + shimpathWow32 += L"\\x86\\renderdocshim32.dll"; + + cmdpathWow32 = renderdocPath; + cmdpathWow32 += L"\\x86\\renderdoccmd.exe"; + } + +#else + + // nothing fancy to do here for 32-bit, just point the shim next to our dll. + *slash = 0; + wcscat_s(renderdocPath, L"\\renderdocshim32.dll"); + shimpathNative = renderdocPath; + +#endif + + GlobalHookData hookdata; + + // try to backup and change the registry settings to start loading our shim dlls. If that fails, + // we bail out immediately + bool success = BackupAndChangeRegistry(hookdata, shimpathWow32, shimpathNative); + if(!success) + return false; PROCESS_INFORMATION pi = {0}; STARTUPINFO si = {0}; @@ -1041,40 +1382,151 @@ void Process::StartGlobalHook(const char *pathmatch, const char *logfile, const std::string debugLogfile = RDCGETLOGFILE(); wstring wdebugLogfile = StringFormat::UTF82Wide(debugLogfile); - _snwprintf_s( - paramsAlloc, 2047, 2047, - L"\"%ls\" globalhook --match \"%ls\" --logfile \"%ls\" --debuglog \"%ls\" --capopts \"%hs\"", - renderdocPath, wpathmatch.c_str(), wlogfile.c_str(), wdebugLogfile.c_str(), optstr.c_str()); + _snwprintf_s(paramsAlloc, 2047, 2047, + L"\"%ls\" globalhook --match \"%ls\" --logfile \"%ls\" --debuglog \"%ls\" " + L"--capopts \"%hs\"", + cmdpathNative.c_str(), wpathmatch.c_str(), wlogfile.c_str(), wdebugLogfile.c_str(), + optstr.c_str()); paramsAlloc[2047] = 0; - BOOL retValue = CreateProcessW(NULL, paramsAlloc, &pSec, &tSec, false, 0, NULL, NULL, &si, &pi); + // we'll be setting stdin + si.dwFlags |= STARTF_USESTDHANDLES; + + // this is the end of the pipe that the child will inherit and use as stdin + HANDLE childEnd = NULL; + + // create a pipe with the writing end for us, and the reading end as the child process's stdin + { + SECURITY_ATTRIBUTES pipeSec; + pipeSec.nLength = sizeof(SECURITY_ATTRIBUTES); + pipeSec.bInheritHandle = TRUE; + pipeSec.lpSecurityDescriptor = NULL; + + BOOL res; + res = CreatePipe(&childEnd, &hookdata.dataNative.pipe, &pipeSec, 0); + + if(!res) + { + RDCERR("Could not create 32-bit stdin pipe"); + RestoreRegistry(hookdata); + return false; + } + + // we don't want the child process to inherit our end + res = SetHandleInformation(hookdata.dataNative.pipe, HANDLE_FLAG_INHERIT, 0); + + if(!res) + { + RDCERR("Could not make 32-bit stdin pipe inheritable"); + RestoreRegistry(hookdata); + return false; + } + + si.hStdInput = childEnd; + } + + // launch the process + BOOL retValue = CreateProcessW(NULL, paramsAlloc, &pSec, &tSec, true, 0, NULL, NULL, &si, &pi); + + // we don't need this end anymore, the child has it + CloseHandle(childEnd); if(retValue == FALSE) - return; + { + RDCERR("Can't launch 64-bit renderdoccmd from '%ls'", cmdpathNative.c_str()); + CloseHandle(hookdata.dataNative.pipe); + RestoreRegistry(hookdata); + return false; + } CloseHandle(pi.hThread); CloseHandle(pi.hProcess); +// repeat the process for the Wow32 renderdoccmd #if ENABLED(RDOC_X64) - *slash = 0; - - wcscat_s(renderdocPath, L"\\x86\\renderdoccmd.exe"); - _snwprintf_s(paramsAlloc, 2047, 2047, - L"\"%ls\" globalhook --match \"%ls\" --log \"%ls\" --capopts \"%hs\"", renderdocPath, - wpathmatch.c_str(), wlogfile.c_str(), optstr.c_str()); + L"\"%ls\" globalhook --match \"%ls\" --log \"%ls\" --capopts \"%hs\"", + cmdpathWow32.c_str(), wpathmatch.c_str(), wlogfile.c_str(), optstr.c_str()); paramsAlloc[2047] = 0; - retValue = CreateProcessW(NULL, paramsAlloc, &pSec, &tSec, false, 0, NULL, NULL, &si, &pi); + { + SECURITY_ATTRIBUTES pipeSec; + pipeSec.nLength = sizeof(SECURITY_ATTRIBUTES); + pipeSec.bInheritHandle = TRUE; + pipeSec.lpSecurityDescriptor = NULL; + + BOOL res; + res = CreatePipe(&childEnd, &hookdata.dataWow32.pipe, &pipeSec, 0); + + if(!res) + { + RDCERR("Could not create 64-bit stdin pipe"); + RestoreRegistry(hookdata); + return false; + } + + res = SetHandleInformation(hookdata.dataWow32.pipe, HANDLE_FLAG_INHERIT, 0); + + if(!res) + { + RDCERR("Could not make 64-bit stdin pipe inheritable"); + RestoreRegistry(hookdata); + return false; + } + + si.hStdInput = childEnd; + } + + retValue = CreateProcessW(NULL, paramsAlloc, &pSec, &tSec, true, 0, NULL, NULL, &si, &pi); + + // we don't need this end anymore + CloseHandle(childEnd); if(retValue == FALSE) - return; + { + RDCERR("Can't launch 32-bit renderdoccmd from '%ls'", cmdpathWow32.c_str()); + CloseHandle(hookdata.dataNative.pipe); + CloseHandle(hookdata.dataWow32.pipe); + RestoreRegistry(hookdata); + return false; + } CloseHandle(pi.hThread); CloseHandle(pi.hProcess); #endif + + // set static global pointer with our data, and launch the thread + globalHook = new GlobalHookData; + *globalHook = hookdata; + + globalHook->pipeThread = Threading::CreateThread(&GlobalHookThread, NULL); + + return true; +} + +bool Process::IsGlobalHookActive() +{ + return globalHook != NULL; +} +void Process::StopGlobalHook() +{ + if(!globalHook) + return; + + // set the finished flag and join to the thread so it closes the pipes (and so the child + // processes) + Atomic::Inc32(&globalHook->finished); + + Threading::JoinThread(globalHook->pipeThread); + Threading::CloseThread(globalHook->pipeThread); + + // restore the registry settings from before we started + RestoreRegistry(*globalHook); + + delete globalHook; + globalHook = NULL; } void *Process::LoadModule(const char *module) diff --git a/renderdoc/replay/entry_points.cpp b/renderdoc/replay/entry_points.cpp index 184fcbba9..e5bf8901b 100644 --- a/renderdoc/replay/entry_points.cpp +++ b/renderdoc/replay/entry_points.cpp @@ -389,11 +389,26 @@ extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_GetDefaultCaptureOptions(Ca *opts = CaptureOptions(); } -extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StartGlobalHook(const char *pathmatch, - const char *logfile, - const CaptureOptions &opts) +extern "C" RENDERDOC_API bool32 RENDERDOC_CC RENDERDOC_StartGlobalHook(const char *pathmatch, + const char *logfile, + const CaptureOptions &opts) { - Process::StartGlobalHook(pathmatch, logfile, opts); + return Process::StartGlobalHook(pathmatch, logfile, opts); +} + +extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StopGlobalHook() +{ + Process::StopGlobalHook(); +} + +extern "C" RENDERDOC_API bool32 RENDERDOC_CC RENDERDOC_IsGlobalHookActive() +{ + return Process::IsGlobalHookActive(); +} + +extern "C" RENDERDOC_API bool32 RENDERDOC_CC RENDERDOC_CanGlobalHook() +{ + return Process::CanGlobalHook(); } extern "C" RENDERDOC_API uint32_t RENDERDOC_CC diff --git a/renderdoccmd/renderdoccmd_win32.cpp b/renderdoccmd/renderdoccmd_win32.cpp index 87671c5ba..b9ce300a5 100644 --- a/renderdoccmd/renderdoccmd_win32.cpp +++ b/renderdoccmd/renderdoccmd_win32.cpp @@ -724,20 +724,12 @@ struct GlobalHookCommand : public Command GetModuleFileNameW(rdoc, rdocpath, _countof(rdocpath) - 1); FreeLibrary(rdoc); - // Create pipe from control program, to stay open until requested to close - HANDLE pipe = CreateFileW( - L"\\\\.\\pipe\\" -#ifdef WIN64 - L"RenderDoc.GlobalHookControl64" -#else - L"RenderDoc.GlobalHookControl32" -#endif - , - GENERIC_READ | GENERIC_WRITE, 0, NULL, OPEN_EXISTING, 0, NULL); + // Create stdin pipe from parent program, to stay open until requested to close + HANDLE pipe = GetStdHandle(STD_INPUT_HANDLE); if(pipe == INVALID_HANDLE_VALUE) { - std::cerr << "globalhook couldn't open control pipe.\n" << std::endl; + std::cerr << "globalhook couldn't open stdin pipe.\n" << std::endl; return 1; } @@ -774,7 +766,7 @@ struct GlobalHookCommand : public Command "ShimData options is too small"); // wait until a write comes in over the pipe - char buf[16]; + char buf[16] = {0}; DWORD read = 0; ReadFile(pipe, buf, 16, &read, NULL); diff --git a/renderdocui/Code/Core.cs b/renderdocui/Code/Core.cs index bc3a474b6..dbc60b7e3 100644 --- a/renderdocui/Code/Core.cs +++ b/renderdocui/Code/Core.cs @@ -59,7 +59,6 @@ namespace renderdocui.Code private bool m_LogLocal = false; private bool m_LogLoaded = false; - private bool m_GlobalHookEnabled = false; private FileSystemWatcher m_LogWatcher = null; @@ -120,8 +119,6 @@ namespace renderdocui.Code public string LogFileName { get { return m_LogFile; } set { if (LogLoaded) m_LogFile = value; } } public bool IsLogLocal { get { return m_LogLocal; } set { m_LogLocal = value; } } - public bool GlobalHookEnabled { get { return m_GlobalHookEnabled; } set { m_GlobalHookEnabled = value; } } - public FetchFrameInfo FrameInfo { get { return m_FrameInfo; } } public APIProperties APIProps { get { return m_APIProperties; } } diff --git a/renderdocui/Interop/StaticExports.cs b/renderdocui/Interop/StaticExports.cs index 02282c287..c6fe4827d 100644 --- a/renderdocui/Interop/StaticExports.cs +++ b/renderdocui/Interop/StaticExports.cs @@ -55,7 +55,13 @@ namespace renderdoc private static extern ReplayCreateStatus RENDERDOC_CreateReplayRenderer(IntPtr logfile, ref float progress, ref IntPtr rendPtr); [DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)] - private static extern void RENDERDOC_StartGlobalHook(IntPtr pathmatch, IntPtr logfile, CaptureOptions opts); + private static extern bool RENDERDOC_StartGlobalHook(IntPtr pathmatch, IntPtr logfile, CaptureOptions opts); + + [DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)] + private static extern bool RENDERDOC_IsGlobalHookActive(); + + [DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)] + private static extern void RENDERDOC_StopGlobalHook(); [DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)] private static extern UInt32 RENDERDOC_ExecuteAndInject(IntPtr app, IntPtr workingDir, IntPtr cmdLine, IntPtr env, @@ -178,15 +184,27 @@ namespace renderdoc return new ReplayRenderer(rendPtr); } - public static void StartGlobalHook(string pathmatch, string logfile, CaptureOptions opts) + public static bool StartGlobalHook(string pathmatch, string logfile, CaptureOptions opts) { IntPtr pathmatch_mem = CustomMarshal.MakeUTF8String(pathmatch); IntPtr logfile_mem = CustomMarshal.MakeUTF8String(logfile); - RENDERDOC_StartGlobalHook(pathmatch_mem, logfile_mem, opts); + bool ret = RENDERDOC_StartGlobalHook(pathmatch_mem, logfile_mem, opts); CustomMarshal.Free(logfile_mem); CustomMarshal.Free(pathmatch_mem); + + return ret; + } + + public static bool IsGlobalHookActive() + { + return RENDERDOC_IsGlobalHookActive(); + } + + public static void StopGlobalHook() + { + RENDERDOC_StopGlobalHook(); } public static UInt32 ExecuteAndInject(string app, string workingDir, string cmdLine, EnvironmentModification[] env, string logfile, CaptureOptions opts) diff --git a/renderdocui/Windows/Dialogs/CaptureDialog.cs b/renderdocui/Windows/Dialogs/CaptureDialog.cs index f0d3e1f4b..749a54fcd 100644 --- a/renderdocui/Windows/Dialogs/CaptureDialog.cs +++ b/renderdocui/Windows/Dialogs/CaptureDialog.cs @@ -801,136 +801,6 @@ namespace renderdocui.Windows.Dialogs m_Core.Config.LastCapturePath = Path.GetDirectoryName(filename); m_Core.Config.LastCaptureExe = Path.GetFileName(filename); } - - private string prevAppInit = ""; - private string prevAppInitWoW64 = ""; - private int prevAppInitEnabled = 0; - private int prevAppInitWoW64Enabled = 0; - - private AutoResetEvent wakeupEvent = new AutoResetEvent(false); - private bool pipeExit = false; - private Thread pipeThread = null; - private NamedPipeServerStream pipe32 = null; - private NamedPipeServerStream pipe64 = null; - - private void EnableAppInit(RegistryKey parent, string path, string dllname, out int prevEnabled, out string prevStr) - { - RegistryKey key = parent.OpenSubKey("Microsoft", true); - if (key == null) { prevEnabled = 0; prevStr = ""; return; } - - key = key.OpenSubKey("Windows NT", true); - if (key == null) { prevEnabled = 0; prevStr = ""; return; } - - key = key.OpenSubKey("CurrentVersion", true); - if (key == null) { prevEnabled = 0; prevStr = ""; return; } - - key = key.OpenSubKey("Windows", true); - if (key == null) { prevEnabled = 0; prevStr = ""; return; } - - object o = key.GetValue("LoadAppInit_DLLs"); - if (o == null || !(o is int)) { prevEnabled = 0; prevStr = ""; return; } - prevEnabled = (int)o; - - o = key.GetValue("AppInit_DLLs"); - if (o == null || !(o is string)) { prevEnabled = 0; prevStr = ""; return; } - prevStr = (string)o; - - key.SetValue("AppInit_DLLs", Win32PInvoke.ShortPath(Path.Combine(path, dllname))); - key.SetValue("LoadAppInit_DLLs", (int)1); - } - - private void RestoreAppInit(RegistryKey parent, int prevEnabled, string prevStr) - { - RegistryKey key = parent.OpenSubKey("Microsoft", true); - if (key == null) { prevEnabled = 0; prevStr = ""; return; } - - key = key.OpenSubKey("Windows NT", true); - if (key == null) { prevEnabled = 0; prevStr = ""; return; } - - key = key.OpenSubKey("CurrentVersion", true); - if (key == null) { prevEnabled = 0; prevStr = ""; return; } - - key = key.OpenSubKey("Windows", true); - if (key == null) { prevEnabled = 0; prevStr = ""; return; } - - key.SetValue("AppInit_DLLs", prevStr); - key.SetValue("LoadAppInit_DLLs", prevEnabled); - } - - private void PipeTick() - { - while (!pipeExit) - { - wakeupEvent.WaitOne(250); - } - - if (pipe32 != null) - { - if (pipe32.IsConnected) - { - using (StreamWriter writer = new StreamWriter(pipe32)) - { - writer.Write("exit"); - writer.Flush(); - } - } - - pipe32.Dispose(); - pipe32 = null; - } - - if (pipe64 != null) - { - if (pipe64.IsConnected) - { - using (StreamWriter writer = new StreamWriter(pipe64)) - { - writer.Write("exit"); - writer.Flush(); - } - } - - pipe64.Dispose(); - pipe64 = null; - } - } - - private void ExitPipeThread() - { - pipeExit = true; - wakeupEvent.Set(); - - if (pipeThread != null) - { - if (pipeThread.ThreadState != ThreadState.Aborted && - pipeThread.ThreadState != ThreadState.Stopped) - { - // try to shut down gracefully - pipeThread.Join(1000); - - if (pipeThread.ThreadState != ThreadState.Aborted && - pipeThread.ThreadState != ThreadState.Stopped) - { - pipeThread.Abort(); - pipeThread.Join(); - } - } - } - - pipeThread = null; - - if (pipe32 != null) - { - pipe32.Dispose(); - pipe32 = null; - } - - if (pipe64 != null) - { - pipe64.Dispose(); - pipe64 = null; - } - } private void toggleGlobalHook_CheckedChanged(object sender, EventArgs e) { @@ -939,8 +809,6 @@ namespace renderdocui.Windows.Dialogs toggleGlobalHook.Enabled = false; - m_Core.GlobalHookEnabled = false; - if (toggleGlobalHook.Checked) { if(!Helpers.IsElevated) @@ -996,98 +864,24 @@ namespace renderdocui.Windows.Dialogs toggleGlobalHook.Text = "Disable Global Hook"; - var path = Path.GetDirectoryName(Path.GetFullPath(Application.ExecutablePath)); + if (StaticExports.IsGlobalHookActive()) + StaticExports.StopGlobalHook(); - var regfile = Path.Combine(Path.GetTempPath(), "RenderDoc_RestoreGlobalHook.reg"); + string exe = exePath.Text; - try - { - if (Environment.Is64BitProcess) - { - EnableAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE").CreateSubKey("Wow6432Node"), - path, "x86\\renderdocshim32.dll", - out prevAppInitWoW64Enabled, out prevAppInitWoW64); + string logfile = exe; + if (logfile.Contains("/")) logfile = logfile.Substring(logfile.LastIndexOf('/') + 1); + if (logfile.Contains("\\")) logfile = logfile.Substring(logfile.LastIndexOf('\\') + 1); + if (logfile.Contains(".")) logfile = logfile.Substring(0, logfile.IndexOf('.')); + logfile = m_Core.TempLogFilename(logfile); - EnableAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE"), - path, "renderdocshim64.dll", - out prevAppInitEnabled, out prevAppInit); + bool success = StaticExports.StartGlobalHook(exe, logfile, GetSettings().Options); - using (FileStream s = File.OpenWrite(regfile)) - { - using (StreamWriter sw = new StreamWriter(s)) - { - sw.WriteLine("Windows Registry Editor Version 5.00"); - sw.WriteLine(""); - sw.WriteLine("[HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows NT\\CurrentVersion\\Windows]"); - sw.WriteLine(String.Format("\"LoadAppInit_DLLs\"=dword:{0:X8}", prevAppInitWoW64Enabled)); - sw.WriteLine(String.Format("\"AppInit_DLLs\"=\"{0}\"", prevAppInitWoW64)); - sw.WriteLine(""); - sw.WriteLine("[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows]"); - sw.WriteLine(String.Format("\"LoadAppInit_DLLs\"=dword:{0:X8}", prevAppInitEnabled)); - sw.WriteLine(String.Format("\"AppInit_DLLs\"=\"{0}\"", prevAppInit)); - sw.Flush(); - } - } - } - else - { - // if this is a 64-bit OS, it will re-direct our request to Wow6432Node anyway, so we - // don't need to handle that manually - EnableAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE"), path, "renderdocshim32.dll", - out prevAppInitEnabled, out prevAppInit); - - using (FileStream s = File.OpenWrite(regfile)) - { - using (StreamWriter sw = new StreamWriter(s)) - { - sw.WriteLine("Windows Registry Editor Version 5.00"); - sw.WriteLine(""); - sw.WriteLine("[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows]"); - sw.WriteLine(String.Format("\"LoadAppInit_DLLs\"=dword:{0:X8}", prevAppInitEnabled)); - sw.WriteLine(String.Format("\"AppInit_DLLs\"=\"{0}\"", prevAppInit)); - sw.Flush(); - } - } - } - } - catch (System.Exception ex) - { - MessageBox.Show("Aborting. Couldn't save backup .reg file to " + regfile + Environment.NewLine + ex.ToString(), "Cannot save registry backup", - MessageBoxButtons.OK, MessageBoxIcon.Error); - - exePath.Enabled = exeBrowse.Enabled = - workDirPath.Enabled = workDirBrowse.Enabled = - cmdline.Enabled = - launch.Enabled = save.Enabled = load.Enabled = true; - - foreach (Control c in capOptsFlow.Controls) - c.Enabled = true; - - foreach (Control c in actionsFlow.Controls) - c.Enabled = true; - - // won't recurse because it's not enabled yet - toggleGlobalHook.Checked = false; - toggleGlobalHook.Text = "Enable Global Hook"; - - toggleGlobalHook.Enabled = true; - return; - } - - ExitPipeThread(); - - pipeExit = false; - - try - { - pipe32 = new NamedPipeServerStream("RenderDoc.GlobalHookControl32"); - pipe64 = new NamedPipeServerStream("RenderDoc.GlobalHookControl64"); - } - catch (System.IO.IOException ex) + if(!success) { // tidy up and exit - MessageBox.Show("Aborting. Couldn't create named pipe:" + Environment.NewLine + ex.Message, - "Cannot create named pipe", + MessageBox.Show("Aborting. Couldn't start global hook. Check diagnostic log in help menu for more information", + "Couldn't start global hook", MessageBoxButtons.OK, MessageBoxIcon.Error); exePath.Enabled = exeBrowse.Enabled = @@ -1101,21 +895,6 @@ namespace renderdocui.Windows.Dialogs foreach (Control c in actionsFlow.Controls) c.Enabled = true; - // need to revert registry entries too - if (Environment.Is64BitProcess) - { - RestoreAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE").CreateSubKey("Wow6432Node"), prevAppInitWoW64Enabled, prevAppInitWoW64); - RestoreAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE"), prevAppInitEnabled, prevAppInit); - } - else - { - // if this is a 64-bit OS, it will re-direct our request to Wow6432Node anyway, so we - // don't need to handle that manually - RestoreAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE"), prevAppInitEnabled, prevAppInit); - } - - if (File.Exists(regfile)) File.Delete(regfile); - // won't recurse because it's not enabled yet toggleGlobalHook.Checked = false; toggleGlobalHook.Text = "Enable Global Hook"; @@ -1123,26 +902,11 @@ namespace renderdocui.Windows.Dialogs toggleGlobalHook.Enabled = true; return; } - - pipeThread = Helpers.NewThread(new ThreadStart(PipeTick)); - - pipeThread.Start(); - - string exe = exePath.Text; - - string logfile = exe; - if (logfile.Contains("/")) logfile = logfile.Substring(logfile.LastIndexOf('/') + 1); - if (logfile.Contains("\\")) logfile = logfile.Substring(logfile.LastIndexOf('\\') + 1); - if (logfile.Contains(".")) logfile = logfile.Substring(0, logfile.IndexOf('.')); - logfile = m_Core.TempLogFilename(logfile); - - StaticExports.StartGlobalHook(exe, logfile, GetSettings().Options); - - m_Core.GlobalHookEnabled = true; } else { - ExitPipeThread(); + if (StaticExports.IsGlobalHookActive()) + StaticExports.StopGlobalHook(); exePath.Enabled = exeBrowse.Enabled = workDirPath.Enabled = workDirBrowse.Enabled = @@ -1156,22 +920,6 @@ namespace renderdocui.Windows.Dialogs c.Enabled = true; toggleGlobalHook.Text = "Enable Global Hook"; - - if (Environment.Is64BitProcess) - { - RestoreAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE").CreateSubKey("Wow6432Node"), prevAppInitWoW64Enabled, prevAppInitWoW64); - RestoreAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE"), prevAppInitEnabled, prevAppInit); - } - else - { - // if this is a 64-bit OS, it will re-direct our request to Wow6432Node anyway, so we - // don't need to handle that manually - RestoreAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE"), prevAppInitEnabled, prevAppInit); - } - - var regfile = Path.Combine(Path.GetTempPath(), "RenderDoc_RestoreGlobalHook.reg"); - - if (File.Exists(regfile)) File.Delete(regfile); } toggleGlobalHook.Enabled = true; diff --git a/renderdocui/Windows/MainWindow.cs b/renderdocui/Windows/MainWindow.cs index 1f4699dc6..3e1e8a574 100644 --- a/renderdocui/Windows/MainWindow.cs +++ b/renderdocui/Windows/MainWindow.cs @@ -1743,7 +1743,7 @@ namespace renderdocui.Windows } } - if (m_Core.GlobalHookEnabled) + if (StaticExports.IsGlobalHookActive()) { MessageBox.Show("Cannot close RenderDoc while global hook is active.", "Global hook active", MessageBoxButtons.OK, MessageBoxIcon.Error);