lee/renderdoc
1
0
Fork 0
mirror of https://github.com/baldurk/renderdoc.git synced 2026-09-22 13:45:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor global hooking, to bring it into the C++ OS-specific code

Browse Source 
* This way it can be used from Qt or any other UI as well.
* The pipes are created internally and just passed as stdin to the
  renderdoccmd processes instead of being named pipes.
This commit is contained in:
baldurk
2017-07-12 12:41:39 +01:00
parent 7fdd770b15
commit 44b74eb813
10 changed files with 586 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files
renderdoc/api/replay/renderdoc_replay.h
+37 -5
View File
@@ -1324,17 +1324,49 @@ DOCUMENT(R"(Retrieve the default and recommended set of capture options.
)");
extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_GetDefaultCaptureOptions(CaptureOptions *opts);
DOCUMENT(R"(Where supported by platform, configuration and setup, begin injecting speculatively into
all new processes started on the system.
DOCUMENT(R"(Begin injecting speculatively into all new processes started on the system. Where
supported by platform, configuration, and setup begin injecting speculatively into all new processes
started on the system.
This function can only be called if global hooking is supported (see :ref:`CanGlobalHook`) and if
global hooking is not active (see :ref:`IsGlobalHookActive`).
This function must be called when the process is running with administrator/superuser permissions.
:param str pathmatch: A string to match against each new process's executable path to determine
which corresponds to the program we actually want to capture.
:param str logfile: Where to store any captures.
:param CaptureOptions opts: The capture options to use when injecting into the program.
:return: ``True`` if the hook is active, ``False`` if something went wrong. The hook must be closed
with :ref:`StopGlobalHook` before the application is closed.
:rtype: ``bool``
)");
extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StartGlobalHook(const char *pathmatch,
const char *logfile,
const CaptureOptions &opts);
extern "C" RENDERDOC_API bool32 RENDERDOC_CC RENDERDOC_StartGlobalHook(const char *pathmatch,
const char *logfile,
const CaptureOptions &opts);
DOCUMENT(R"(Stop the global hook that was activated by :ref:`StartGlobalHook`.
This function can only be called if global hooking is supported (see :ref:`CanGlobalHook`) and if
global hooking is active (see :ref:`IsGlobalHookActive`).
)");
extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StopGlobalHook();
DOCUMENT(R"(Determines if the global hook is active or not.
This function can only be called if global hooking is supported (see :ref:`CanGlobalHook`).
:return: ``True`` if the hook is active, or ``False`` if the hook is inactive.
:rtype: ``bool``
)");
extern "C" RENDERDOC_API bool32 RENDERDOC_CC RENDERDOC_IsGlobalHookActive();
DOCUMENT(R"(Determines if the global hook is supported on the current platform and configuration.
:return: ``True`` if global hooking can be used on the platform, ``False`` if not.
:rtype: ``bool``
)");
extern "C" RENDERDOC_API bool32 RENDERDOC_CC RENDERDOC_CanGlobalHook();
DOCUMENT(R"(Launch an application and inject into it to allow capturing.
renderdoc/os/os_specific.h
+5 -1
View File
@@ -52,7 +52,11 @@ void RegisterEnvironmentModification(EnvironmentModification modif);
void ApplyEnvironmentModification();
void StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions &opts);
bool CanGlobalHook();
bool StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions &opts);
bool IsGlobalHookActive();
void StopGlobalHook();
uint32_t InjectIntoProcess(uint32_t pid, const rdctype::array<EnvironmentModification> &env,
const char *logfile, const CaptureOptions &opts, bool waitForExit);
struct ProcessResult
renderdoc/os/posix/posix_process.cpp
+16 -1
View File
@@ -601,9 +601,24 @@ uint32_t Process::LaunchAndInjectIntoProcess(const char *app, const char *workin
return ret;
}
void Process::StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions &opts)
bool Process::StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions &opts)
{
RDCUNIMPLEMENTED("Global hooking of all processes on linux");
return false;
}
bool Process::CanGlobalHook()
{
return false;
}
bool Process::IsGlobalHookActive()
{
return false;
}
void Process::StopGlobalHook()
{
}
void *Process::LoadModule(const char *module)
renderdoc/os/win32/win32_process.cpp
+469 -17
View File
@@ -996,10 +996,276 @@ uint32_t Process::LaunchAndInjectIntoProcess(const char *app, const char *workin
return ret;
}
void Process::StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions &opts)
bool Process::CanGlobalHook()
{
// all we need is admin rights and it's the caller's responsibility to ensure that.
return true;
}
// to simplify the below code, rather than splitting by 32-bit/64-bit we split by native and Wow32.
// This means that for 32-bit code (whether it's on 32-bit OS or not) we just have native, and the
// Wow32 stuff is empty/unused. For 64-bit we use both. Thus the native registry key is always the
// same path regardless of the bitness we're running as and we don't have to move things around or
// have conditionals all over
struct GlobalHookData
{
struct
{
HANDLE pipe = NULL;
DWORD appinitEnabled = 0;
wstring appinitDLLs;
} dataNative, dataWow32;
volatile int32_t finished = 0;
Threading::ThreadHandle pipeThread = 0;
};
// utility function to close the registry keys, print an error, and quit
static bool HandleRegError(HKEY keyNative, HKEY keyWow32, LSTATUS ret, const char *msg)
{
if(keyNative)
RegCloseKey(keyNative);
if(keyWow32)
RegCloseKey(keyWow32);
RDCERR("Error with AppInit registry keys - %s (%d)", msg, ret);
return false;
}
#define REG_CHECK(msg) \
if(ret != ERROR_SUCCESS) \
{ \
return HandleRegError(keyNative, keyWow32, ret, msg); \
}
// function to backup the previous settings for AppInit, then enable it and write our own paths.
bool BackupAndChangeRegistry(GlobalHookData &hookdata, const wstring &shimpathWow32,
const wstring &shimpathNative)
{
HKEY keyNative = NULL;
HKEY keyWow32 = NULL;
// open the native key
LSTATUS ret = RegCreateKeyExA(HKEY_LOCAL_MACHINE,
"SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows", 0, NULL,
0, KEY_READ | KEY_WRITE, NULL, &keyNative, NULL);
REG_CHECK("Could not open AppInit key");
// if we are doing Wow32, open that key as well
if(!shimpathWow32.empty())
{
ret = RegCreateKeyExA(HKEY_LOCAL_MACHINE,
"SOFTWARE\\Wow6432Node\\Microsoft\\Windows NT\\CurrentVersion\\Windows",
0, NULL, 0, KEY_READ | KEY_WRITE, NULL, &keyWow32, NULL);
REG_CHECK("Could not open AppInit key");
}
const DWORD one = 1;
// fetch the previous data for LoadAppInit_DLLs and AppInit_DLLs
DWORD sz = 4;
ret = RegGetValueA(keyNative, NULL, "LoadAppInit_DLLs", RRF_RT_REG_DWORD, NULL,
(void *)&hookdata.dataNative.appinitEnabled, &sz);
REG_CHECK("Could not fetch LoadAppInit_DLLs");
sz = 0;
ret = RegGetValueW(keyNative, NULL, L"AppInit_DLLs", RRF_RT_ANY, NULL, NULL, &sz);
if(ret == ERROR_MORE_DATA || ret == ERROR_SUCCESS)
{
hookdata.dataNative.appinitDLLs.resize(sz / sizeof(wchar_t));
ret = RegGetValueW(keyNative, NULL, L"AppInit_DLLs", RRF_RT_ANY, NULL,
(void *)&hookdata.dataNative.appinitDLLs[0], &sz);
}
REG_CHECK("Could not fetch AppInit_DLLs");
// set DWORD:1 for LoadAppInit_DLLs and convert our path to a short path then set it
ret = RegSetValueExA(keyNative, "LoadAppInit_DLLs", 0, REG_DWORD, (const BYTE *)&one, sizeof(one));
REG_CHECK("Could not set LoadAppInit_DLLs");
wstring shortpath;
shortpath = shimpathNative;
GetShortPathNameW(shimpathNative.c_str(), (wchar_t *)&shortpath[0], (DWORD)shortpath.size());
ret = RegSetValueExW(keyNative, L"AppInit_DLLs", 0, REG_SZ, (const BYTE *)shortpath.data(),
DWORD(shortpath.size() * sizeof(wchar_t)));
REG_CHECK("Could not set AppInit_DLLs");
// if we're doing Wow32, repeat the process for those keys
if(keyWow32)
{
sz = 4;
ret = RegGetValueA(keyWow32, NULL, "LoadAppInit_DLLs", RRF_RT_REG_DWORD, NULL,
(void *)&hookdata.dataWow32.appinitEnabled, &sz);
REG_CHECK("Could not fetch LoadAppInit_DLLs");
sz = 0;
ret = RegGetValueW(keyWow32, NULL, L"AppInit_DLLs", RRF_RT_ANY, NULL, NULL, &sz);
if(ret == ERROR_MORE_DATA || ret == ERROR_SUCCESS)
{
hookdata.dataWow32.appinitDLLs.resize(sz / sizeof(wchar_t));
ret = RegGetValueW(keyWow32, NULL, L"AppInit_DLLs", RRF_RT_ANY, NULL,
(void *)&hookdata.dataWow32.appinitDLLs[0], &sz);
}
REG_CHECK("Could not fetch AppInit_DLLs");
ret = RegSetValueExA(keyWow32, "LoadAppInit_DLLs", 0, REG_DWORD, (const BYTE *)&one, sizeof(one));
REG_CHECK("Could not set LoadAppInit_DLLs");
shortpath = shimpathWow32;
GetShortPathNameW(shimpathWow32.c_str(), &shortpath[0], (DWORD)shortpath.size());
ret = RegSetValueExW(keyWow32, L"AppInit_DLLs", 0, REG_SZ, (const BYTE *)shortpath.data(),
DWORD(shortpath.size() * sizeof(wchar_t)));
REG_CHECK("Could not set AppInit_DLLs");
}
wstring backup;
// write a .reg file that contains the previous settings, so that if all else fails the user can
// manually insert it back into the registry to restore everything.
backup += L"Windows Registry Editor Version 5.00\n";
backup += L"\n";
backup += L"[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows]\n";
backup += L"\"LoadAppInit_DLLs\"=dword:0000000";
backup += (hookdata.dataNative.appinitEnabled ? L"1\n" : L"0\n");
backup += L"\"AppInit_DLLs\"=\"";
// we append with the C string so we don't add trailing NULLs into the text.
backup += hookdata.dataNative.appinitDLLs.c_str();
backup += L"\"\n";
if(keyWow32)
{
backup += L"\n";
backup +=
L"[HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\Microsoft\\"
L"Windows NT\\CurrentVersion\\Windows]\n";
backup += L"\"LoadAppInit_DLLs\"=dword:0000000";
backup += (hookdata.dataWow32.appinitEnabled ? L"1\n" : L"0\n");
backup += L"\"AppInit_DLLs\"=\"";
backup += hookdata.dataWow32.appinitDLLs.c_str();
backup += L"\"\n";
}
if(keyNative)
RegCloseKey(keyNative);
if(keyWow32)
RegCloseKey(keyWow32);
keyNative = keyWow32 = NULL;
// write it to disk but don't fail if we can't, just print it to the log and keep going.
wchar_t reg_backup[MAX_PATH];
GetTempPathW(MAX_PATH, reg_backup);
wcscat_s(reg_backup, L"RenderDoc_RestoreGlobalHook.reg");
FILE *f = NULL;
_wfopen_s(&f, reg_backup, L"w");
if(f)
{
fputws(backup.c_str(), f);
fclose(f);
}
else
{
RDCERR("Error opening registry backup file %ls", reg_backup);
RDCERR("Backup registry data is:\n\n%ls\n\n", backup.c_str());
}
return true;
}
// switch error-handling to print-and-continue, as we can't really do anything about it at this
// point and we want to continue restoring in case only one thing failed.
#undef REG_CHECK
#define REG_CHECK(msg) \
if(ret != ERROR_SUCCESS) \
{ \
HandleRegError(keyNative, keyWow32, ret, "Could not open AppInit key"); \
}
void RestoreRegistry(const GlobalHookData &hookdata)
{
HKEY keyNative = NULL;
HKEY keyWow32 = NULL;
LSTATUS ret = RegCreateKeyExA(HKEY_LOCAL_MACHINE,
"SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows", 0, NULL,
0, KEY_READ | KEY_WRITE, NULL, &keyNative, NULL);
REG_CHECK("Could not open AppInit key");
#if ENABLED(RDOC_X64)
ret = RegCreateKeyExA(HKEY_LOCAL_MACHINE,
"SOFTWARE\\Wow6432Node\\Microsoft\\Windows NT\\CurrentVersion\\Windows", 0,
NULL, 0, KEY_READ | KEY_WRITE, NULL, &keyWow32, NULL);
REG_CHECK("Could not open AppInit key");
#endif
// set the native values back to where they were
ret = RegSetValueExA(keyNative, "LoadAppInit_DLLs", 0, REG_DWORD,
(const BYTE *)&hookdata.dataNative.appinitEnabled,
sizeof(hookdata.dataNative.appinitEnabled));
REG_CHECK("Could not set LoadAppInit_DLLs");
ret = RegSetValueExW(keyNative, L"AppInit_DLLs", 0, REG_SZ,
(const BYTE *)hookdata.dataNative.appinitDLLs.data(),
DWORD(hookdata.dataNative.appinitDLLs.size() * sizeof(wchar_t)));
REG_CHECK("Could not set AppInit_DLLs");
// if we opened it, restore the Wow32 values as well
if(keyWow32)
{
ret = RegSetValueExA(keyWow32, "LoadAppInit_DLLs", 0, REG_DWORD,
(const BYTE *)&hookdata.dataWow32.appinitEnabled,
sizeof(hookdata.dataWow32.appinitEnabled));
REG_CHECK("Could not set LoadAppInit_DLLs");
ret = RegSetValueExW(keyWow32, L"AppInit_DLLs", 0, REG_SZ,
(const BYTE *)hookdata.dataWow32.appinitDLLs.data(),
DWORD(hookdata.dataWow32.appinitDLLs.size() * sizeof(wchar_t)));
REG_CHECK("Could not set AppInit_DLLs");
}
}
static GlobalHookData *globalHook = NULL;
// a thread we run in the background just to keep the pipes open and wait until we're ready to stop
// the global hook.
static void GlobalHookThread(void *)
{
// keep looping doing an atomic compare-exchange to check that finished is still 0
while(Atomic::CmpExch32(&globalHook->finished, 0, 0) == 0)
{
// wake every quarter of a second to test again
Threading::Sleep(250);
}
char exitData[32] = "exit";
// write some data into the pipe and close it. The data is (currently) unimportant, just that it
// causes the blocking read on the other end to succeed and close the program.
DWORD dummy = 0;
if(globalHook->dataNative.pipe)
{
WriteFile(globalHook->dataNative.pipe, exitData, (DWORD)sizeof(exitData), &dummy, NULL);
CloseHandle(globalHook->dataNative.pipe);
}
if(globalHook->dataWow32.pipe)
{
WriteFile(globalHook->dataWow32.pipe, exitData, (DWORD)sizeof(exitData), &dummy, NULL);
CloseHandle(globalHook->dataWow32.pipe);
}
}
bool Process::StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions &opts)
{
if(pathmatch == NULL)
return;
return false;
wchar_t renderdocPath[MAX_PATH] = {0};
GetModuleFileNameW(GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll"), &renderdocPath[0],
@@ -1012,7 +1278,82 @@ void Process::StartGlobalHook(const char *pathmatch, const char *logfile, const
else
slash = renderdocPath + wcslen(renderdocPath);
wcscat_s(renderdocPath, L"\\renderdoccmd.exe");
// the native renderdoccmd.exe is always next to the dll. Wow32 will be somewhere else
wstring cmdpathNative = renderdocPath;
cmdpathNative += L"\\renderdoccmd.exe";
wstring cmdpathWow32;
wstring shimpathNative = renderdocPath;
wstring shimpathWow32;
#if ENABLED(RDOC_X64)
// native shim is just renderdocshim64.dll
*slash = 0;
wcscat_s(renderdocPath, L"\\renderdocshim64.dll");
shimpathNative = renderdocPath;
// if it looks like we're in the development environment, look for the alternate bitness in the
// corresponding folder
const wchar_t *devLocation = wcsstr(renderdocPath, L"\\x64\\Development\\");
if(devLocation)
{
size_t idx = devLocation - renderdocPath;
renderdocPath[idx] = 0;
shimpathWow32 = renderdocPath;
shimpathWow32 += L"\\Win32\\Development\\renderdocshim32.dll";
cmdpathWow32 = renderdocPath;
cmdpathWow32 += L"\\Win32\\Development\\renderdoccmd.exe";
}
if(!devLocation)
{
devLocation = wcsstr(renderdocPath, L"\\x64\\Release\\");
if(devLocation)
{
size_t idx = devLocation - renderdocPath;
renderdocPath[idx] = 0;
shimpathWow32 = renderdocPath;
shimpathWow32 += L"\\Win32\\Release\\renderdocshim32.dll";
cmdpathWow32 = renderdocPath;
cmdpathWow32 += L"\\Win32\\Release\\renderdoccmd.exe";
}
}
// if we're not in the dev environment, assume it's under a x86\ subfolder
if(!devLocation)
{
*slash = 0;
shimpathWow32 = renderdocPath;
shimpathWow32 += L"\\x86\\renderdocshim32.dll";
cmdpathWow32 = renderdocPath;
cmdpathWow32 += L"\\x86\\renderdoccmd.exe";
}
#else
// nothing fancy to do here for 32-bit, just point the shim next to our dll.
*slash = 0;
wcscat_s(renderdocPath, L"\\renderdocshim32.dll");
shimpathNative = renderdocPath;
#endif
GlobalHookData hookdata;
// try to backup and change the registry settings to start loading our shim dlls. If that fails,
// we bail out immediately
bool success = BackupAndChangeRegistry(hookdata, shimpathWow32, shimpathNative);
if(!success)
return false;
PROCESS_INFORMATION pi = {0};
STARTUPINFO si = {0};
@@ -1041,40 +1382,151 @@ void Process::StartGlobalHook(const char *pathmatch, const char *logfile, const
std::string debugLogfile = RDCGETLOGFILE();
wstring wdebugLogfile = StringFormat::UTF82Wide(debugLogfile);
_snwprintf_s(
paramsAlloc, 2047, 2047,
L"\"%ls\" globalhook --match \"%ls\" --logfile \"%ls\" --debuglog \"%ls\" --capopts \"%hs\"",
renderdocPath, wpathmatch.c_str(), wlogfile.c_str(), wdebugLogfile.c_str(), optstr.c_str());
_snwprintf_s(paramsAlloc, 2047, 2047,
L"\"%ls\" globalhook --match \"%ls\" --logfile \"%ls\" --debuglog \"%ls\" "
L"--capopts \"%hs\"",
cmdpathNative.c_str(), wpathmatch.c_str(), wlogfile.c_str(), wdebugLogfile.c_str(),
optstr.c_str());
paramsAlloc[2047] = 0;
BOOL retValue = CreateProcessW(NULL, paramsAlloc, &pSec, &tSec, false, 0, NULL, NULL, &si, &pi);
// we'll be setting stdin
si.dwFlags |= STARTF_USESTDHANDLES;
// this is the end of the pipe that the child will inherit and use as stdin
HANDLE childEnd = NULL;
// create a pipe with the writing end for us, and the reading end as the child process's stdin
{
SECURITY_ATTRIBUTES pipeSec;
pipeSec.nLength = sizeof(SECURITY_ATTRIBUTES);
pipeSec.bInheritHandle = TRUE;
pipeSec.lpSecurityDescriptor = NULL;
BOOL res;
res = CreatePipe(&childEnd, &hookdata.dataNative.pipe, &pipeSec, 0);
if(!res)
{
RDCERR("Could not create 32-bit stdin pipe");
RestoreRegistry(hookdata);
return false;
}
// we don't want the child process to inherit our end
res = SetHandleInformation(hookdata.dataNative.pipe, HANDLE_FLAG_INHERIT, 0);
if(!res)
{
RDCERR("Could not make 32-bit stdin pipe inheritable");
RestoreRegistry(hookdata);
return false;
}
si.hStdInput = childEnd;
}
// launch the process
BOOL retValue = CreateProcessW(NULL, paramsAlloc, &pSec, &tSec, true, 0, NULL, NULL, &si, &pi);
// we don't need this end anymore, the child has it
CloseHandle(childEnd);
if(retValue == FALSE)
return;
{
RDCERR("Can't launch 64-bit renderdoccmd from '%ls'", cmdpathNative.c_str());
CloseHandle(hookdata.dataNative.pipe);
RestoreRegistry(hookdata);
return false;
}
CloseHandle(pi.hThread);
CloseHandle(pi.hProcess);
// repeat the process for the Wow32 renderdoccmd
#if ENABLED(RDOC_X64)
*slash = 0;
wcscat_s(renderdocPath, L"\\x86\\renderdoccmd.exe");
_snwprintf_s(paramsAlloc, 2047, 2047,
L"\"%ls\" globalhook --match \"%ls\" --log \"%ls\" --capopts \"%hs\"", renderdocPath,
wpathmatch.c_str(), wlogfile.c_str(), optstr.c_str());
L"\"%ls\" globalhook --match \"%ls\" --log \"%ls\" --capopts \"%hs\"",
cmdpathWow32.c_str(), wpathmatch.c_str(), wlogfile.c_str(), optstr.c_str());
paramsAlloc[2047] = 0;
retValue = CreateProcessW(NULL, paramsAlloc, &pSec, &tSec, false, 0, NULL, NULL, &si, &pi);
{
SECURITY_ATTRIBUTES pipeSec;
pipeSec.nLength = sizeof(SECURITY_ATTRIBUTES);
pipeSec.bInheritHandle = TRUE;
pipeSec.lpSecurityDescriptor = NULL;
BOOL res;
res = CreatePipe(&childEnd, &hookdata.dataWow32.pipe, &pipeSec, 0);
if(!res)
{
RDCERR("Could not create 64-bit stdin pipe");
RestoreRegistry(hookdata);
return false;
}
res = SetHandleInformation(hookdata.dataWow32.pipe, HANDLE_FLAG_INHERIT, 0);
if(!res)
{
RDCERR("Could not make 64-bit stdin pipe inheritable");
RestoreRegistry(hookdata);
return false;
}
si.hStdInput = childEnd;
}
retValue = CreateProcessW(NULL, paramsAlloc, &pSec, &tSec, true, 0, NULL, NULL, &si, &pi);
// we don't need this end anymore
CloseHandle(childEnd);
if(retValue == FALSE)
return;
{
RDCERR("Can't launch 32-bit renderdoccmd from '%ls'", cmdpathWow32.c_str());
CloseHandle(hookdata.dataNative.pipe);
CloseHandle(hookdata.dataWow32.pipe);
RestoreRegistry(hookdata);
return false;
}
CloseHandle(pi.hThread);
CloseHandle(pi.hProcess);
#endif
// set static global pointer with our data, and launch the thread
globalHook = new GlobalHookData;
*globalHook = hookdata;
globalHook->pipeThread = Threading::CreateThread(&GlobalHookThread, NULL);
return true;
}
bool Process::IsGlobalHookActive()
{
return globalHook != NULL;
}
void Process::StopGlobalHook()
{
if(!globalHook)
return;
// set the finished flag and join to the thread so it closes the pipes (and so the child
// processes)
Atomic::Inc32(&globalHook->finished);
Threading::JoinThread(globalHook->pipeThread);
Threading::CloseThread(globalHook->pipeThread);
// restore the registry settings from before we started
RestoreRegistry(*globalHook);
delete globalHook;
globalHook = NULL;
}
void *Process::LoadModule(const char *module)
renderdoc/replay/entry_points.cpp
+19 -4
View File
@@ -389,11 +389,26 @@ extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_GetDefaultCaptureOptions(Ca
*opts = CaptureOptions();
}
extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StartGlobalHook(const char *pathmatch,
const char *logfile,
const CaptureOptions &opts)
extern "C" RENDERDOC_API bool32 RENDERDOC_CC RENDERDOC_StartGlobalHook(const char *pathmatch,
const char *logfile,
const CaptureOptions &opts)
{
Process::StartGlobalHook(pathmatch, logfile, opts);
return Process::StartGlobalHook(pathmatch, logfile, opts);
}
extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StopGlobalHook()
{
Process::StopGlobalHook();
}
extern "C" RENDERDOC_API bool32 RENDERDOC_CC RENDERDOC_IsGlobalHookActive()
{
return Process::IsGlobalHookActive();
}
extern "C" RENDERDOC_API bool32 RENDERDOC_CC RENDERDOC_CanGlobalHook()
{
return Process::CanGlobalHook();
}
extern "C" RENDERDOC_API uint32_t RENDERDOC_CC
renderdoccmd/renderdoccmd_win32.cpp
+4 -12
View File
@@ -724,20 +724,12 @@ struct GlobalHookCommand : public Command
GetModuleFileNameW(rdoc, rdocpath, _countof(rdocpath) - 1);
FreeLibrary(rdoc);
// Create pipe from control program, to stay open until requested to close
HANDLE pipe = CreateFileW(
L"\\\\.\\pipe\\"
#ifdef WIN64
L"RenderDoc.GlobalHookControl64"
#else
L"RenderDoc.GlobalHookControl32"
#endif
,
GENERIC_READ | GENERIC_WRITE, 0, NULL, OPEN_EXISTING, 0, NULL);
// Create stdin pipe from parent program, to stay open until requested to close
HANDLE pipe = GetStdHandle(STD_INPUT_HANDLE);
if(pipe == INVALID_HANDLE_VALUE)
{
std::cerr << "globalhook couldn't open control pipe.\n" << std::endl;
std::cerr << "globalhook couldn't open stdin pipe.\n" << std::endl;
return 1;
}
@@ -774,7 +766,7 @@ struct GlobalHookCommand : public Command
"ShimData options is too small");
// wait until a write comes in over the pipe
char buf[16];
char buf[16] = {0};
DWORD read = 0;
ReadFile(pipe, buf, 16, &read, NULL);
renderdocui/Code/Core.cs
-3
View File
@@ -59,7 +59,6 @@ namespace renderdocui.Code
private bool m_LogLocal = false;
private bool m_LogLoaded = false;
private bool m_GlobalHookEnabled = false;
private FileSystemWatcher m_LogWatcher = null;
@@ -120,8 +119,6 @@ namespace renderdocui.Code
public string LogFileName { get { return m_LogFile; } set { if (LogLoaded) m_LogFile = value; } }
public bool IsLogLocal { get { return m_LogLocal; } set { m_LogLocal = value; } }
public bool GlobalHookEnabled { get { return m_GlobalHookEnabled; } set { m_GlobalHookEnabled = value; } }
public FetchFrameInfo FrameInfo { get { return m_FrameInfo; } }
public APIProperties APIProps { get { return m_APIProperties; } }
renderdocui/Interop/StaticExports.cs
+21 -3
View File
@@ -55,7 +55,13 @@ namespace renderdoc
private static extern ReplayCreateStatus RENDERDOC_CreateReplayRenderer(IntPtr logfile, ref float progress, ref IntPtr rendPtr);
[DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
private static extern void RENDERDOC_StartGlobalHook(IntPtr pathmatch, IntPtr logfile, CaptureOptions opts);
private static extern bool RENDERDOC_StartGlobalHook(IntPtr pathmatch, IntPtr logfile, CaptureOptions opts);
[DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
private static extern bool RENDERDOC_IsGlobalHookActive();
[DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
private static extern void RENDERDOC_StopGlobalHook();
[DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
private static extern UInt32 RENDERDOC_ExecuteAndInject(IntPtr app, IntPtr workingDir, IntPtr cmdLine, IntPtr env,
@@ -178,15 +184,27 @@ namespace renderdoc
return new ReplayRenderer(rendPtr);
}
public static void StartGlobalHook(string pathmatch, string logfile, CaptureOptions opts)
public static bool StartGlobalHook(string pathmatch, string logfile, CaptureOptions opts)
{
IntPtr pathmatch_mem = CustomMarshal.MakeUTF8String(pathmatch);
IntPtr logfile_mem = CustomMarshal.MakeUTF8String(logfile);
RENDERDOC_StartGlobalHook(pathmatch_mem, logfile_mem, opts);
bool ret = RENDERDOC_StartGlobalHook(pathmatch_mem, logfile_mem, opts);
CustomMarshal.Free(logfile_mem);
CustomMarshal.Free(pathmatch_mem);
return ret;
}
public static bool IsGlobalHookActive()
{
return RENDERDOC_IsGlobalHookActive();
}
public static void StopGlobalHook()
{
RENDERDOC_StopGlobalHook();
}
public static UInt32 ExecuteAndInject(string app, string workingDir, string cmdLine, EnvironmentModification[] env, string logfile, CaptureOptions opts)
renderdocui/Windows/Dialogs/CaptureDialog.cs
+14 -266
View File
@@ -801,136 +801,6 @@ namespace renderdocui.Windows.Dialogs
m_Core.Config.LastCapturePath = Path.GetDirectoryName(filename);
m_Core.Config.LastCaptureExe = Path.GetFileName(filename);
}
private string prevAppInit = "";
private string prevAppInitWoW64 = "";
private int prevAppInitEnabled = 0;
private int prevAppInitWoW64Enabled = 0;
private AutoResetEvent wakeupEvent = new AutoResetEvent(false);
private bool pipeExit = false;
private Thread pipeThread = null;
private NamedPipeServerStream pipe32 = null;
private NamedPipeServerStream pipe64 = null;
private void EnableAppInit(RegistryKey parent, string path, string dllname, out int prevEnabled, out string prevStr)
{
RegistryKey key = parent.OpenSubKey("Microsoft", true);
if (key == null) { prevEnabled = 0; prevStr = ""; return; }
key = key.OpenSubKey("Windows NT", true);
if (key == null) { prevEnabled = 0; prevStr = ""; return; }
key = key.OpenSubKey("CurrentVersion", true);
if (key == null) { prevEnabled = 0; prevStr = ""; return; }
key = key.OpenSubKey("Windows", true);
if (key == null) { prevEnabled = 0; prevStr = ""; return; }
object o = key.GetValue("LoadAppInit_DLLs");
if (o == null || !(o is int)) { prevEnabled = 0; prevStr = ""; return; }
prevEnabled = (int)o;
o = key.GetValue("AppInit_DLLs");
if (o == null || !(o is string)) { prevEnabled = 0; prevStr = ""; return; }
prevStr = (string)o;
key.SetValue("AppInit_DLLs", Win32PInvoke.ShortPath(Path.Combine(path, dllname)));
key.SetValue("LoadAppInit_DLLs", (int)1);
}
private void RestoreAppInit(RegistryKey parent, int prevEnabled, string prevStr)
{
RegistryKey key = parent.OpenSubKey("Microsoft", true);
if (key == null) { prevEnabled = 0; prevStr = ""; return; }
key = key.OpenSubKey("Windows NT", true);
if (key == null) { prevEnabled = 0; prevStr = ""; return; }
key = key.OpenSubKey("CurrentVersion", true);
if (key == null) { prevEnabled = 0; prevStr = ""; return; }
key = key.OpenSubKey("Windows", true);
if (key == null) { prevEnabled = 0; prevStr = ""; return; }
key.SetValue("AppInit_DLLs", prevStr);
key.SetValue("LoadAppInit_DLLs", prevEnabled);
}
private void PipeTick()
{
while (!pipeExit)
{
wakeupEvent.WaitOne(250);
}
if (pipe32 != null)
{
if (pipe32.IsConnected)
{
using (StreamWriter writer = new StreamWriter(pipe32))
{
writer.Write("exit");
writer.Flush();
}
}
pipe32.Dispose();
pipe32 = null;
}
if (pipe64 != null)
{
if (pipe64.IsConnected)
{
using (StreamWriter writer = new StreamWriter(pipe64))
{
writer.Write("exit");
writer.Flush();
}
}
pipe64.Dispose();
pipe64 = null;
}
}
private void ExitPipeThread()
{
pipeExit = true;
wakeupEvent.Set();
if (pipeThread != null)
{
if (pipeThread.ThreadState != ThreadState.Aborted &&
pipeThread.ThreadState != ThreadState.Stopped)
{
// try to shut down gracefully
pipeThread.Join(1000);
if (pipeThread.ThreadState != ThreadState.Aborted &&
pipeThread.ThreadState != ThreadState.Stopped)
{
pipeThread.Abort();
pipeThread.Join();
}
}
}
pipeThread = null;
if (pipe32 != null)
{
pipe32.Dispose();
pipe32 = null;
}
if (pipe64 != null)
{
pipe64.Dispose();
pipe64 = null;
}
}
private void toggleGlobalHook_CheckedChanged(object sender, EventArgs e)
{
@@ -939,8 +809,6 @@ namespace renderdocui.Windows.Dialogs
toggleGlobalHook.Enabled = false;
m_Core.GlobalHookEnabled = false;
if (toggleGlobalHook.Checked)
{
if(!Helpers.IsElevated)
@@ -996,98 +864,24 @@ namespace renderdocui.Windows.Dialogs
toggleGlobalHook.Text = "Disable Global Hook";
var path = Path.GetDirectoryName(Path.GetFullPath(Application.ExecutablePath));
if (StaticExports.IsGlobalHookActive())
StaticExports.StopGlobalHook();
var regfile = Path.Combine(Path.GetTempPath(), "RenderDoc_RestoreGlobalHook.reg");
string exe = exePath.Text;
try
{
if (Environment.Is64BitProcess)
{
EnableAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE").CreateSubKey("Wow6432Node"),
path, "x86\\renderdocshim32.dll",
out prevAppInitWoW64Enabled, out prevAppInitWoW64);
string logfile = exe;
if (logfile.Contains("/")) logfile = logfile.Substring(logfile.LastIndexOf('/') + 1);
if (logfile.Contains("\\")) logfile = logfile.Substring(logfile.LastIndexOf('\\') + 1);
if (logfile.Contains(".")) logfile = logfile.Substring(0, logfile.IndexOf('.'));
logfile = m_Core.TempLogFilename(logfile);
EnableAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE"),
path, "renderdocshim64.dll",
out prevAppInitEnabled, out prevAppInit);
bool success = StaticExports.StartGlobalHook(exe, logfile, GetSettings().Options);
using (FileStream s = File.OpenWrite(regfile))
{
using (StreamWriter sw = new StreamWriter(s))
{
sw.WriteLine("Windows Registry Editor Version 5.00");
sw.WriteLine("");
sw.WriteLine("[HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows NT\\CurrentVersion\\Windows]");
sw.WriteLine(String.Format("\"LoadAppInit_DLLs\"=dword:{0:X8}", prevAppInitWoW64Enabled));
sw.WriteLine(String.Format("\"AppInit_DLLs\"=\"{0}\"", prevAppInitWoW64));
sw.WriteLine("");
sw.WriteLine("[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows]");
sw.WriteLine(String.Format("\"LoadAppInit_DLLs\"=dword:{0:X8}", prevAppInitEnabled));
sw.WriteLine(String.Format("\"AppInit_DLLs\"=\"{0}\"", prevAppInit));
sw.Flush();
}
}
}
else
{
// if this is a 64-bit OS, it will re-direct our request to Wow6432Node anyway, so we
// don't need to handle that manually
EnableAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE"), path, "renderdocshim32.dll",
out prevAppInitEnabled, out prevAppInit);
using (FileStream s = File.OpenWrite(regfile))
{
using (StreamWriter sw = new StreamWriter(s))
{
sw.WriteLine("Windows Registry Editor Version 5.00");
sw.WriteLine("");
sw.WriteLine("[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows]");
sw.WriteLine(String.Format("\"LoadAppInit_DLLs\"=dword:{0:X8}", prevAppInitEnabled));
sw.WriteLine(String.Format("\"AppInit_DLLs\"=\"{0}\"", prevAppInit));
sw.Flush();
}
}
}
}
catch (System.Exception ex)
{
MessageBox.Show("Aborting. Couldn't save backup .reg file to " + regfile + Environment.NewLine + ex.ToString(), "Cannot save registry backup",
MessageBoxButtons.OK, MessageBoxIcon.Error);
exePath.Enabled = exeBrowse.Enabled =
workDirPath.Enabled = workDirBrowse.Enabled =
cmdline.Enabled =
launch.Enabled = save.Enabled = load.Enabled = true;
foreach (Control c in capOptsFlow.Controls)
c.Enabled = true;
foreach (Control c in actionsFlow.Controls)
c.Enabled = true;
// won't recurse because it's not enabled yet
toggleGlobalHook.Checked = false;
toggleGlobalHook.Text = "Enable Global Hook";
toggleGlobalHook.Enabled = true;
return;
}
ExitPipeThread();
pipeExit = false;
try
{
pipe32 = new NamedPipeServerStream("RenderDoc.GlobalHookControl32");
pipe64 = new NamedPipeServerStream("RenderDoc.GlobalHookControl64");
}
catch (System.IO.IOException ex)
if(!success)
{
// tidy up and exit
MessageBox.Show("Aborting. Couldn't create named pipe:" + Environment.NewLine + ex.Message,
"Cannot create named pipe",
MessageBox.Show("Aborting. Couldn't start global hook. Check diagnostic log in help menu for more information",
"Couldn't start global hook",
MessageBoxButtons.OK, MessageBoxIcon.Error);
exePath.Enabled = exeBrowse.Enabled =
@@ -1101,21 +895,6 @@ namespace renderdocui.Windows.Dialogs
foreach (Control c in actionsFlow.Controls)
c.Enabled = true;
// need to revert registry entries too
if (Environment.Is64BitProcess)
{
RestoreAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE").CreateSubKey("Wow6432Node"), prevAppInitWoW64Enabled, prevAppInitWoW64);
RestoreAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE"), prevAppInitEnabled, prevAppInit);
}
else
{
// if this is a 64-bit OS, it will re-direct our request to Wow6432Node anyway, so we
// don't need to handle that manually
RestoreAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE"), prevAppInitEnabled, prevAppInit);
}
if (File.Exists(regfile)) File.Delete(regfile);
// won't recurse because it's not enabled yet
toggleGlobalHook.Checked = false;
toggleGlobalHook.Text = "Enable Global Hook";
@@ -1123,26 +902,11 @@ namespace renderdocui.Windows.Dialogs
toggleGlobalHook.Enabled = true;
return;
}
pipeThread = Helpers.NewThread(new ThreadStart(PipeTick));
pipeThread.Start();
string exe = exePath.Text;
string logfile = exe;
if (logfile.Contains("/")) logfile = logfile.Substring(logfile.LastIndexOf('/') + 1);
if (logfile.Contains("\\")) logfile = logfile.Substring(logfile.LastIndexOf('\\') + 1);
if (logfile.Contains(".")) logfile = logfile.Substring(0, logfile.IndexOf('.'));
logfile = m_Core.TempLogFilename(logfile);
StaticExports.StartGlobalHook(exe, logfile, GetSettings().Options);
m_Core.GlobalHookEnabled = true;
}
else
{
ExitPipeThread();
if (StaticExports.IsGlobalHookActive())
StaticExports.StopGlobalHook();
exePath.Enabled = exeBrowse.Enabled =
workDirPath.Enabled = workDirBrowse.Enabled =
@@ -1156,22 +920,6 @@ namespace renderdocui.Windows.Dialogs
c.Enabled = true;
toggleGlobalHook.Text = "Enable Global Hook";
if (Environment.Is64BitProcess)
{
RestoreAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE").CreateSubKey("Wow6432Node"), prevAppInitWoW64Enabled, prevAppInitWoW64);
RestoreAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE"), prevAppInitEnabled, prevAppInit);
}
else
{
// if this is a 64-bit OS, it will re-direct our request to Wow6432Node anyway, so we
// don't need to handle that manually
RestoreAppInit(Registry.LocalMachine.CreateSubKey("SOFTWARE"), prevAppInitEnabled, prevAppInit);
}
var regfile = Path.Combine(Path.GetTempPath(), "RenderDoc_RestoreGlobalHook.reg");
if (File.Exists(regfile)) File.Delete(regfile);
}
toggleGlobalHook.Enabled = true;
renderdocui/Windows/MainWindow.cs
+1 -1
View File
@@ -1743,7 +1743,7 @@ namespace renderdocui.Windows
}
}
if (m_Core.GlobalHookEnabled)
if (StaticExports.IsGlobalHookActive())
{
MessageBox.Show("Cannot close RenderDoc while global hook is active.", "Global hook active",
MessageBoxButtons.OK, MessageBoxIcon.Error);