lee/renderdoc
1
0
Fork 0
mirror of https://github.com/baldurk/renderdoc.git synced 2026-05-05 01:20:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Don't enable global hook if short paths are unavailable. Closes #2645

Browse Source 
* AppInit_DLLs requires short paths to be specified, if they can't be created on
  the volume we can't reliably enable the global hook.
* Detecting this programmatically is quite complex, but since we know the shim
  file is always longer than 8.3, if we detect that the short path is the same
  length as the long path we know short names weren't used.
This commit is contained in:
baldurk
2022-07-08 11:21:44 +01:00
parent 92971fb061
commit 83bff08fd1
1 changed files with 40 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
renderdoc/os/win32/win32_process.cpp
+40 -9
View File
@@ -1192,7 +1192,7 @@ struct GlobalHookData
};
// utility function to close the registry keys, print an error, and quit
static bool HandleRegError(HKEY keyNative, HKEY keyWow32, LSTATUS ret, const char *msg)
static RDResult HandleRegError(HKEY keyNative, HKEY keyWow32, LSTATUS ret, const char *msg)
{
if(keyNative)
RegCloseKey(keyNative);
@@ -1200,8 +1200,11 @@ static bool HandleRegError(HKEY keyNative, HKEY keyWow32, LSTATUS ret, const cha
if(keyWow32)
RegCloseKey(keyWow32);
RDCERR("Error with AppInit registry keys - %s (%d)", msg, ret);
return false;
RDCLOG("Error with AppInit registry keys - %s (%d)", msg, ret);
RETURN_ERROR_RESULT(ResultCode::InjectionFailed,
"Error updating registry to enable global hook.\n"
"Check that RenderDoc is correctly running as administrator.");
}
#define REG_CHECK(msg) \
@@ -1211,12 +1214,40 @@ static bool HandleRegError(HKEY keyNative, HKEY keyWow32, LSTATUS ret, const cha
}
// function to backup the previous settings for AppInit, then enable it and write our own paths.
bool BackupAndChangeRegistry(GlobalHookData &hookdata, const rdcstr &shimpathWow32,
const rdcstr &shimpathNative)
RDResult BackupAndChangeRegistry(GlobalHookData &hookdata, const rdcstr &shimpathWow32,
const rdcstr &shimpathNative)
{
HKEY keyNative = NULL;
HKEY keyWow32 = NULL;
// AppInit_DLLs requires short paths, but short paths can be disabled globally or on a per-volume
// level. If short paths are disabled we'll get the long path back, we *always* expect the path to
// get shorter because the shim filename is bigger than 8.3.
DWORD nativeShortSize = GetShortPathNameW(StringFormat::UTF82Wide(shimpathNative).c_str(), NULL,
(DWORD)shimpathNative.length());
if(nativeShortSize == (DWORD)shimpathNative.length() + 1)
{
RETURN_ERROR_RESULT(
ResultCode::FileIOFailed,
"RenderDoc is installed on a volume or system that has short paths disabled.\n"
"For the global hook, short paths must be enabled where RenderDoc is installed.");
}
if(!shimpathWow32.empty())
{
DWORD wow32ShortSize = GetShortPathNameW(StringFormat::UTF82Wide(shimpathWow32).c_str(), NULL,
(DWORD)shimpathWow32.length());
if(wow32ShortSize == (DWORD)shimpathWow32.length() + 1)
{
RETURN_ERROR_RESULT(
ResultCode::FileIOFailed,
"RenderDoc is installed on a volume or system that has short paths disabled.\n"
"For the global hook, short paths must be enabled where RenderDoc is installed.");
}
}
// open the native key
LSTATUS ret = RegCreateKeyExA(HKEY_LOCAL_MACHINE,
"SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows", 0, NULL,
@@ -1346,7 +1377,7 @@ bool BackupAndChangeRegistry(GlobalHookData &hookdata, const rdcstr &shimpathWow
RDCERR("Backup registry data is:\n\n%ls\n\n", backup.c_str());
}
return true;
return RDResult();
}
// switch error-handling to print-and-continue, as we can't really do anything about it at this
@@ -1502,9 +1533,9 @@ RDResult Process::StartGlobalHook(const rdcstr &pathmatch, const rdcstr &capture
// try to backup and change the registry settings to start loading our shim dlls. If that fails,
// we bail out immediately
bool success = BackupAndChangeRegistry(hookdata, shimpathWow32, shimpathNative);
if(!success)
return RDResult(ResultCode::InternalError, "Registry change failed");
RDResult regStatus = BackupAndChangeRegistry(hookdata, shimpathWow32, shimpathNative);
if(regStatus != ResultCode::Succeeded)
return regStatus;
PROCESS_INFORMATION pi = {0};
STARTUPINFO si = {0};