mirror of
https://github.com/baldurk/renderdoc.git
synced 2026-05-05 01:20:42 +00:00
2c80a7206b
* Mostly we add an explicit default: break but in some cases we add explicit cases and/or error messages.
1640 lines
47 KiB
C++
1640 lines
47 KiB
C++
/******************************************************************************
|
|
* The MIT License (MIT)
|
|
*
|
|
* Copyright (c) 2015-2019 Baldur Karlsson
|
|
* Copyright (c) 2014 Crytek
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
* in the Software without restriction, including without limitation the rights
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
* furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
* THE SOFTWARE.
|
|
******************************************************************************/
|
|
|
|
// must be separate so that it's included first and not sorted by clang-format
|
|
#include <windows.h>
|
|
|
|
#include <Psapi.h>
|
|
#include <tchar.h>
|
|
#include <tlhelp32.h>
|
|
#include "common/formatting.h"
|
|
#include "core/core.h"
|
|
#include "os/os_specific.h"
|
|
#include "strings/string_utils.h"
|
|
|
|
static rdcarray<EnvironmentModification> &GetEnvModifications()
|
|
{
|
|
static rdcarray<EnvironmentModification> envCallbacks;
|
|
return envCallbacks;
|
|
}
|
|
|
|
struct InsensitiveComparison
|
|
{
|
|
bool operator()(const rdcstr &a, const rdcstr &b) const { return strlower(a) < strlower(b); }
|
|
};
|
|
|
|
typedef std::map<rdcstr, rdcstr, InsensitiveComparison> EnvMap;
|
|
|
|
static EnvMap EnvStringToEnvMap(const wchar_t *envstring)
|
|
{
|
|
EnvMap ret;
|
|
|
|
const wchar_t *e = envstring;
|
|
|
|
while(*e)
|
|
{
|
|
const wchar_t *equals = wcschr(e, L'=');
|
|
|
|
rdcstr name = StringFormat::Wide2UTF8(rdcwstr(e, equals - e));
|
|
rdcstr value = StringFormat::Wide2UTF8(equals + 1);
|
|
|
|
ret[name] = value;
|
|
|
|
// jump to \0 and past it
|
|
e += wcslen(e) + 1;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
void Process::RegisterEnvironmentModification(const EnvironmentModification &modif)
|
|
{
|
|
GetEnvModifications().push_back(modif);
|
|
}
|
|
|
|
static void ApplyEnvModifications(EnvMap &envValues,
|
|
const rdcarray<EnvironmentModification> &modifications,
|
|
bool setToSystem)
|
|
{
|
|
for(size_t i = 0; i < modifications.size(); i++)
|
|
{
|
|
const EnvironmentModification &m = modifications[i];
|
|
|
|
rdcstr value;
|
|
|
|
auto it = envValues.find(m.name);
|
|
if(it != envValues.end())
|
|
value = it->second;
|
|
|
|
switch(m.mod)
|
|
{
|
|
case EnvMod::Set: value = m.value.c_str(); break;
|
|
case EnvMod::Append:
|
|
{
|
|
if(!value.empty())
|
|
{
|
|
if(m.sep == EnvSep::Platform || m.sep == EnvSep::SemiColon)
|
|
value += ";";
|
|
else if(m.sep == EnvSep::Colon)
|
|
value += ":";
|
|
}
|
|
value += m.value.c_str();
|
|
break;
|
|
}
|
|
case EnvMod::Prepend:
|
|
{
|
|
if(!value.empty())
|
|
{
|
|
rdcstr prep = m.value;
|
|
if(m.sep == EnvSep::Platform || m.sep == EnvSep::SemiColon)
|
|
prep += ";";
|
|
else if(m.sep == EnvSep::Colon)
|
|
prep += ":";
|
|
value = prep + value;
|
|
}
|
|
else
|
|
{
|
|
value = m.value.c_str();
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
envValues[m.name] = value;
|
|
|
|
if(setToSystem)
|
|
SetEnvironmentVariableW(StringFormat::UTF82Wide(m.name).c_str(),
|
|
StringFormat::UTF82Wide(value).c_str());
|
|
}
|
|
}
|
|
|
|
// on windows we apply environment changes here, after process initialisation
|
|
// but before any real work (in RenderDoc::Initialise) so that we support
|
|
// injecting the dll into processes we didn't launch (ie didn't control the
|
|
// starting environment for), or even the application loading the dll itself
|
|
// without any interaction with our replay app.
|
|
void Process::ApplyEnvironmentModification()
|
|
{
|
|
// turn environment string to a UTF-8 map
|
|
LPWCH envStrings = GetEnvironmentStringsW();
|
|
EnvMap envValues = EnvStringToEnvMap(envStrings);
|
|
FreeEnvironmentStringsW(envStrings);
|
|
rdcarray<EnvironmentModification> &modifications = GetEnvModifications();
|
|
|
|
ApplyEnvModifications(envValues, modifications, true);
|
|
|
|
// these have been applied to the current process
|
|
modifications.clear();
|
|
}
|
|
|
|
const char *Process::GetEnvVariable(const char *name)
|
|
{
|
|
DWORD ret = GetEnvironmentVariableA(name, NULL, 0);
|
|
if(ret == 0 && GetLastError() == ERROR_ENVVAR_NOT_FOUND)
|
|
return NULL;
|
|
|
|
static char buf[1024] = {};
|
|
if(ret >= 1024)
|
|
RDCERR("Static buffer insufficiently sized");
|
|
|
|
RDCEraseEl(buf);
|
|
GetEnvironmentVariableA(name, buf, RDCMIN((DWORD)1023U, ret));
|
|
return buf;
|
|
}
|
|
|
|
uint64_t Process::GetMemoryUsage()
|
|
{
|
|
HANDLE proc = GetCurrentProcess();
|
|
|
|
if(proc == NULL)
|
|
{
|
|
RDCERR("Couldn't open process: %d", GetLastError());
|
|
return 0;
|
|
}
|
|
|
|
PROCESS_MEMORY_COUNTERS memInfo = {};
|
|
|
|
uint64_t ret = 0;
|
|
|
|
if(GetProcessMemoryInfo(proc, &memInfo, sizeof(memInfo)))
|
|
{
|
|
ret = memInfo.WorkingSetSize;
|
|
}
|
|
else
|
|
{
|
|
RDCERR("Couldn't get process memory info: %d", GetLastError());
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
// helpers for various shims and dlls etc, not part of the public API
|
|
extern "C" __declspec(dllexport) void __cdecl INTERNAL_GetTargetControlIdent(uint32_t *ident)
|
|
{
|
|
if(ident)
|
|
*ident = RenderDoc::Inst().GetTargetControlIdent();
|
|
}
|
|
|
|
extern "C" __declspec(dllexport) void __cdecl INTERNAL_SetCaptureOptions(CaptureOptions *opts)
|
|
{
|
|
if(opts)
|
|
RenderDoc::Inst().SetCaptureOptions(*opts);
|
|
}
|
|
|
|
extern "C" __declspec(dllexport) void __cdecl INTERNAL_SetCaptureFile(const char *capfile)
|
|
{
|
|
if(capfile)
|
|
RenderDoc::Inst().SetCaptureFileTemplate(capfile);
|
|
}
|
|
|
|
static EnvironmentModification tempEnvMod;
|
|
|
|
extern "C" __declspec(dllexport) void __cdecl INTERNAL_EnvModName(const char *name)
|
|
{
|
|
if(name)
|
|
tempEnvMod.name = name;
|
|
}
|
|
|
|
extern "C" __declspec(dllexport) void __cdecl INTERNAL_EnvModValue(const char *value)
|
|
{
|
|
if(value)
|
|
tempEnvMod.value = value;
|
|
}
|
|
|
|
extern "C" __declspec(dllexport) void __cdecl INTERNAL_EnvSep(EnvSep *sep)
|
|
{
|
|
if(sep)
|
|
tempEnvMod.sep = *sep;
|
|
}
|
|
|
|
extern "C" __declspec(dllexport) void __cdecl INTERNAL_EnvMod(EnvMod *mod)
|
|
{
|
|
if(mod)
|
|
{
|
|
tempEnvMod.mod = *mod;
|
|
Process::RegisterEnvironmentModification(tempEnvMod);
|
|
}
|
|
}
|
|
|
|
extern "C" __declspec(dllexport) void __cdecl INTERNAL_ApplyEnvMods(void *ignored)
|
|
{
|
|
Process::ApplyEnvironmentModification();
|
|
}
|
|
|
|
void InjectDLL(HANDLE hProcess, rdcwstr libName)
|
|
{
|
|
wchar_t dllPath[MAX_PATH + 1] = {0};
|
|
wcscpy_s(dllPath, libName.c_str());
|
|
|
|
static HMODULE kernel32 = GetModuleHandleA("kernel32.dll");
|
|
|
|
if(kernel32 == NULL)
|
|
{
|
|
RDCERR("Couldn't get handle for kernel32.dll");
|
|
return;
|
|
}
|
|
|
|
void *remoteMem =
|
|
VirtualAllocEx(hProcess, NULL, sizeof(dllPath), MEM_COMMIT, PAGE_EXECUTE_READWRITE);
|
|
if(remoteMem)
|
|
{
|
|
BOOL success = WriteProcessMemory(hProcess, remoteMem, (void *)dllPath, sizeof(dllPath), NULL);
|
|
if(success)
|
|
{
|
|
HANDLE hThread = CreateRemoteThread(
|
|
hProcess, NULL, 0, (LPTHREAD_START_ROUTINE)GetProcAddress(kernel32, "LoadLibraryW"),
|
|
remoteMem, 0, NULL);
|
|
if(hThread)
|
|
{
|
|
WaitForSingleObject(hThread, INFINITE);
|
|
CloseHandle(hThread);
|
|
}
|
|
else
|
|
{
|
|
RDCERR("Couldn't create remote thread for LoadLibraryW: %u", GetLastError());
|
|
}
|
|
}
|
|
else
|
|
{
|
|
RDCERR("Couldn't write remote memory %p with dllPath '%ls': %u", remoteMem, dllPath,
|
|
GetLastError());
|
|
}
|
|
|
|
VirtualFreeEx(hProcess, remoteMem, 0, MEM_RELEASE);
|
|
}
|
|
else
|
|
{
|
|
RDCERR("Couldn't allocate remote memory for DLL '%ls': %u", libName.c_str(), GetLastError());
|
|
}
|
|
}
|
|
|
|
uintptr_t FindRemoteDLL(DWORD pid, rdcstr libName)
|
|
{
|
|
HANDLE hModuleSnap = INVALID_HANDLE_VALUE;
|
|
|
|
rdcwstr wlibName = StringFormat::UTF82Wide(strlower(libName));
|
|
|
|
// up to 10 retries
|
|
for(int i = 0; i < 10; i++)
|
|
{
|
|
hModuleSnap = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE, pid);
|
|
|
|
if(hModuleSnap == INVALID_HANDLE_VALUE)
|
|
{
|
|
DWORD err = GetLastError();
|
|
|
|
RDCWARN("CreateToolhelp32Snapshot(%u) -> 0x%08x", pid, err);
|
|
|
|
// retry if error is ERROR_BAD_LENGTH
|
|
if(err == ERROR_BAD_LENGTH)
|
|
continue;
|
|
}
|
|
|
|
// didn't retry, or succeeded
|
|
break;
|
|
}
|
|
|
|
if(hModuleSnap == INVALID_HANDLE_VALUE)
|
|
{
|
|
RDCERR("Couldn't create toolhelp dump of modules in process %u", pid);
|
|
return 0;
|
|
}
|
|
|
|
MODULEENTRY32 me32;
|
|
RDCEraseEl(me32);
|
|
me32.dwSize = sizeof(MODULEENTRY32);
|
|
|
|
BOOL success = Module32First(hModuleSnap, &me32);
|
|
|
|
if(success == FALSE)
|
|
{
|
|
DWORD err = GetLastError();
|
|
|
|
RDCERR("Couldn't get first module in process %u: 0x%08x", pid, err);
|
|
CloseHandle(hModuleSnap);
|
|
return 0;
|
|
}
|
|
|
|
uintptr_t ret = 0;
|
|
|
|
int numModules = 0;
|
|
|
|
do
|
|
{
|
|
wchar_t modnameLower[MAX_MODULE_NAME32 + 1];
|
|
RDCEraseEl(modnameLower);
|
|
wcsncpy_s(modnameLower, me32.szModule, MAX_MODULE_NAME32);
|
|
|
|
wchar_t *wc = &modnameLower[0];
|
|
while(*wc)
|
|
{
|
|
*wc = towlower(*wc);
|
|
wc++;
|
|
}
|
|
|
|
numModules++;
|
|
|
|
if(wcsstr(modnameLower, wlibName.c_str()) == modnameLower)
|
|
{
|
|
ret = (uintptr_t)me32.modBaseAddr;
|
|
}
|
|
} while(ret == 0 && Module32Next(hModuleSnap, &me32));
|
|
|
|
if(ret == 0)
|
|
{
|
|
HANDLE h = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, pid);
|
|
|
|
DWORD exitCode = 0;
|
|
|
|
if(h)
|
|
GetExitCodeProcess(h, &exitCode);
|
|
|
|
if(h == NULL || exitCode != STILL_ACTIVE)
|
|
{
|
|
RDCERR(
|
|
"Error injecting into remote process with PID %u which is no longer available.\n"
|
|
"Possibly the process has crashed during early startup, or is missing DLLs to run?",
|
|
pid);
|
|
}
|
|
else
|
|
{
|
|
RDCERR("Couldn't find module '%s' among %d modules", libName.c_str(), numModules);
|
|
}
|
|
|
|
if(h)
|
|
CloseHandle(h);
|
|
}
|
|
|
|
CloseHandle(hModuleSnap);
|
|
|
|
return ret;
|
|
}
|
|
|
|
void InjectFunctionCall(HANDLE hProcess, uintptr_t renderdoc_remote, const char *funcName,
|
|
void *data, const size_t dataLen)
|
|
{
|
|
if(dataLen == 0)
|
|
{
|
|
RDCERR("Invalid function call injection attempt");
|
|
return;
|
|
}
|
|
|
|
RDCDEBUG("Injecting call to %s", funcName);
|
|
|
|
HMODULE renderdoc_local = GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll");
|
|
|
|
uintptr_t func_local = (uintptr_t)GetProcAddress(renderdoc_local, funcName);
|
|
|
|
// we've found SetCaptureOptions in our local instance of the module, now calculate the offset and
|
|
// so get the function
|
|
// in the remote module (which might be loaded at a different base address
|
|
uintptr_t func_remote = func_local + renderdoc_remote - (uintptr_t)renderdoc_local;
|
|
|
|
void *remoteMem = VirtualAllocEx(hProcess, NULL, dataLen, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
|
|
SIZE_T numWritten;
|
|
WriteProcessMemory(hProcess, remoteMem, data, dataLen, &numWritten);
|
|
|
|
HANDLE hThread =
|
|
CreateRemoteThread(hProcess, NULL, 0, (LPTHREAD_START_ROUTINE)func_remote, remoteMem, 0, NULL);
|
|
WaitForSingleObject(hThread, INFINITE);
|
|
|
|
ReadProcessMemory(hProcess, remoteMem, data, dataLen, &numWritten);
|
|
|
|
CloseHandle(hThread);
|
|
VirtualFreeEx(hProcess, remoteMem, 0, MEM_RELEASE);
|
|
}
|
|
|
|
static PROCESS_INFORMATION RunProcess(const char *app, const char *workingDir, const char *cmdLine,
|
|
const rdcarray<EnvironmentModification> &env, bool internal,
|
|
HANDLE *phChildStdOutput_Rd, HANDLE *phChildStdError_Rd)
|
|
{
|
|
PROCESS_INFORMATION pi;
|
|
STARTUPINFO si;
|
|
SECURITY_ATTRIBUTES pSec;
|
|
SECURITY_ATTRIBUTES tSec;
|
|
|
|
RDCEraseEl(pi);
|
|
RDCEraseEl(si);
|
|
RDCEraseEl(pSec);
|
|
RDCEraseEl(tSec);
|
|
|
|
pSec.nLength = sizeof(pSec);
|
|
tSec.nLength = sizeof(tSec);
|
|
|
|
rdcwstr workdir = L"";
|
|
|
|
if(workingDir != NULL && workingDir[0] != 0)
|
|
workdir = StringFormat::UTF82Wide(workingDir);
|
|
else
|
|
workdir = StringFormat::UTF82Wide(get_dirname(app));
|
|
|
|
wchar_t *paramsAlloc = NULL;
|
|
|
|
rdcwstr wapp = StringFormat::UTF82Wide(app);
|
|
|
|
// CreateProcessW can modify the params, need space.
|
|
size_t len = wapp.length() + 10;
|
|
|
|
rdcwstr wcmd = L"";
|
|
|
|
if(cmdLine != NULL && cmdLine[0] != 0)
|
|
{
|
|
wcmd = StringFormat::UTF82Wide(rdcstr(cmdLine));
|
|
len += wcmd.length();
|
|
}
|
|
|
|
paramsAlloc = new wchar_t[len];
|
|
|
|
RDCEraseMem(paramsAlloc, len * sizeof(wchar_t));
|
|
|
|
wcscpy_s(paramsAlloc, len, L"\"");
|
|
wcscat_s(paramsAlloc, len, wapp.c_str());
|
|
wcscat_s(paramsAlloc, len, L"\"");
|
|
|
|
if(cmdLine != NULL && cmdLine[0] != 0)
|
|
{
|
|
wcscat_s(paramsAlloc, len, L" ");
|
|
wcscat_s(paramsAlloc, len, wcmd.c_str());
|
|
}
|
|
|
|
HANDLE hChildStdOutput_Wr = 0, hChildStdError_Wr = 0;
|
|
if(phChildStdOutput_Rd)
|
|
{
|
|
RDCASSERT(phChildStdError_Rd);
|
|
|
|
SECURITY_ATTRIBUTES sa;
|
|
sa.nLength = sizeof(SECURITY_ATTRIBUTES);
|
|
sa.bInheritHandle = TRUE;
|
|
sa.lpSecurityDescriptor = NULL;
|
|
|
|
if(!CreatePipe(phChildStdOutput_Rd, &hChildStdOutput_Wr, &sa, 0))
|
|
RDCERR("Could not create pipe to read stdout");
|
|
if(!SetHandleInformation(*phChildStdOutput_Rd, HANDLE_FLAG_INHERIT, 0))
|
|
RDCERR("Could not set pipe handle information");
|
|
|
|
if(!CreatePipe(phChildStdError_Rd, &hChildStdError_Wr, &sa, 0))
|
|
RDCERR("Could not create pipe to read stdout");
|
|
if(!SetHandleInformation(*phChildStdError_Rd, HANDLE_FLAG_INHERIT, 0))
|
|
RDCERR("Could not set pipe handle information");
|
|
|
|
si.dwFlags |= STARTF_USESTDHANDLES;
|
|
si.hStdOutput = hChildStdOutput_Wr;
|
|
si.hStdError = hChildStdError_Wr;
|
|
}
|
|
|
|
// if it's a utility launch, hide the command prompt window from showing
|
|
if(phChildStdOutput_Rd || internal)
|
|
si.dwFlags |= STARTF_USESHOWWINDOW;
|
|
|
|
if(!internal)
|
|
RDCLOG("Running process %s", app);
|
|
|
|
// turn environment string to a UTF-8 map
|
|
std::wstring envString;
|
|
|
|
if(!env.empty())
|
|
{
|
|
LPWCH envStrings = GetEnvironmentStringsW();
|
|
EnvMap envValues = EnvStringToEnvMap(envStrings);
|
|
FreeEnvironmentStringsW(envStrings);
|
|
|
|
ApplyEnvModifications(envValues, env, false);
|
|
|
|
for(auto it = envValues.begin(); it != envValues.end(); ++it)
|
|
{
|
|
envString += StringFormat::UTF82Wide(it->first).c_str();
|
|
envString += L"=";
|
|
envString += StringFormat::UTF82Wide(it->second).c_str();
|
|
envString.push_back(0);
|
|
}
|
|
}
|
|
|
|
BOOL retValue = CreateProcessW(NULL, paramsAlloc, &pSec, &tSec,
|
|
true, // Need to inherit handles for ReadFile to read stdout
|
|
CREATE_SUSPENDED | CREATE_UNICODE_ENVIRONMENT,
|
|
envString.empty() ? NULL : (void *)envString.data(),
|
|
workdir.c_str(), &si, &pi);
|
|
|
|
if(phChildStdOutput_Rd)
|
|
{
|
|
CloseHandle(hChildStdOutput_Wr);
|
|
CloseHandle(hChildStdError_Wr);
|
|
}
|
|
|
|
SAFE_DELETE_ARRAY(paramsAlloc);
|
|
|
|
if(!retValue)
|
|
{
|
|
if(!internal)
|
|
RDCWARN("Process %s could not be loaded.", app);
|
|
CloseHandle(pi.hProcess);
|
|
CloseHandle(pi.hThread);
|
|
RDCEraseEl(pi);
|
|
}
|
|
|
|
return pi;
|
|
}
|
|
|
|
rdcpair<ReplayStatus, uint32_t> Process::InjectIntoProcess(
|
|
uint32_t pid, const rdcarray<EnvironmentModification> &env, const char *capturefile,
|
|
const CaptureOptions &opts, bool waitForExit)
|
|
{
|
|
rdcwstr wcapturefile = capturefile == NULL ? L"" : StringFormat::UTF82Wide(capturefile);
|
|
|
|
HANDLE hProcess =
|
|
OpenProcess(PROCESS_CREATE_THREAD | PROCESS_QUERY_INFORMATION | PROCESS_VM_OPERATION |
|
|
PROCESS_VM_WRITE | PROCESS_VM_READ | SYNCHRONIZE,
|
|
FALSE, pid);
|
|
|
|
if(opts.delayForDebugger > 0)
|
|
{
|
|
RDCDEBUG("Waiting for debugger attach to %lu", pid);
|
|
uint32_t timeout = 0;
|
|
|
|
BOOL debuggerAttached = FALSE;
|
|
|
|
while(!debuggerAttached)
|
|
{
|
|
CheckRemoteDebuggerPresent(hProcess, &debuggerAttached);
|
|
|
|
Sleep(10);
|
|
timeout += 10;
|
|
|
|
if(timeout > opts.delayForDebugger * 1000)
|
|
break;
|
|
}
|
|
|
|
if(debuggerAttached)
|
|
RDCDEBUG("Debugger attach detected after %.2f s", float(timeout) / 1000.0f);
|
|
else
|
|
RDCDEBUG("Timed out waiting for debugger, gave up after %u s", opts.delayForDebugger);
|
|
}
|
|
|
|
RDCLOG("Injecting renderdoc into process %lu", pid);
|
|
|
|
wchar_t renderdocPath[MAX_PATH] = {0};
|
|
GetModuleFileNameW(GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll"), &renderdocPath[0],
|
|
MAX_PATH - 1);
|
|
|
|
wchar_t renderdocPathLower[MAX_PATH] = {0};
|
|
memcpy(renderdocPathLower, renderdocPath, MAX_PATH * sizeof(wchar_t));
|
|
for(size_t i = 0; i < MAX_PATH && renderdocPathLower[i]; i++)
|
|
{
|
|
// lowercase
|
|
if(renderdocPathLower[i] >= 'A' && renderdocPathLower[i] <= 'Z')
|
|
renderdocPathLower[i] = 'a' + char(renderdocPathLower[i] - 'A');
|
|
|
|
// normalise paths
|
|
if(renderdocPathLower[i] == '/')
|
|
renderdocPathLower[i] = '\\';
|
|
}
|
|
|
|
BOOL isWow64 = FALSE;
|
|
BOOL success = IsWow64Process(hProcess, &isWow64);
|
|
|
|
if(!success)
|
|
{
|
|
DWORD err = GetLastError();
|
|
RDCERR("Couldn't determine bitness of process, err: %08x", err);
|
|
CloseHandle(hProcess);
|
|
return {ReplayStatus::IncompatibleProcess, 0};
|
|
}
|
|
|
|
bool capalt = false;
|
|
|
|
#if DISABLED(RDOC_X64)
|
|
BOOL selfWow64 = FALSE;
|
|
|
|
HANDLE hSelfProcess = GetCurrentProcess();
|
|
|
|
// check to see if we're a WoW64 process
|
|
success = IsWow64Process(hSelfProcess, &selfWow64);
|
|
|
|
CloseHandle(hSelfProcess);
|
|
|
|
if(!success)
|
|
{
|
|
DWORD err = GetLastError();
|
|
RDCERR("Couldn't determine bitness of self, err: %08x", err);
|
|
return {ReplayStatus::InternalError, 0};
|
|
}
|
|
|
|
// we know we're 32-bit, so if the target process is not wow64
|
|
// and we are, it's 64-bit. If we're both not wow64 then we're
|
|
// running on 32-bit windows, and if we're both wow64 then we're
|
|
// both 32-bit on 64-bit windows.
|
|
//
|
|
// We don't support capturing 64-bit programs from a 32-bit install
|
|
// because it's pointless - a 64-bit install will work for all in
|
|
// that case. But we do want to handle the case of:
|
|
// 64-bit renderdoc -> 32-bit program (via 32-bit renderdoccmd)
|
|
// -> 64-bit program (going back to 64-bit renderdoccmd).
|
|
// so we try to see if we're an x86 invoked renderdoccmd in an
|
|
// otherwise 64-bit install, and 'promote' back to 64-bit.
|
|
if(selfWow64 && !isWow64)
|
|
{
|
|
wchar_t *slash = wcsrchr(renderdocPath, L'\\');
|
|
|
|
if(slash && slash > renderdocPath + 4)
|
|
{
|
|
slash -= 4;
|
|
|
|
if(slash && !wcsncmp(slash, L"\\x86", 4))
|
|
{
|
|
RDCDEBUG("Promoting back to 64-bit");
|
|
capalt = true;
|
|
}
|
|
}
|
|
|
|
// if it looks like we're in the development environment, look for the alternate bitness in the
|
|
// corresponding folder
|
|
if(!capalt)
|
|
{
|
|
const wchar_t *devLocation = wcsstr(renderdocPathLower, L"\\win32\\development\\");
|
|
if(!devLocation)
|
|
devLocation = wcsstr(renderdocPathLower, L"\\win32\\release\\");
|
|
|
|
if(devLocation)
|
|
{
|
|
RDCDEBUG("Promoting back to 64-bit");
|
|
capalt = true;
|
|
}
|
|
}
|
|
|
|
// if we couldn't promote, then bail out.
|
|
if(!capalt)
|
|
{
|
|
RDCDEBUG("Running from %ls", renderdocPathLower);
|
|
RDCERR("Can't capture x64 process with x86 renderdoc");
|
|
|
|
CloseHandle(hProcess);
|
|
return {ReplayStatus::IncompatibleProcess, 0};
|
|
}
|
|
}
|
|
#else
|
|
// farm off to alternate bitness renderdoccmd.exe
|
|
|
|
// if the target process is 'wow64' that means it's 32-bit.
|
|
capalt = (isWow64 == TRUE);
|
|
#endif
|
|
|
|
if(capalt)
|
|
{
|
|
#if ENABLED(RDOC_X64)
|
|
// if it looks like we're in the development environment, look for the alternate bitness in the
|
|
// corresponding folder
|
|
const wchar_t *devLocation = wcsstr(renderdocPathLower, L"\\x64\\development\\");
|
|
if(devLocation)
|
|
{
|
|
size_t idx = devLocation - renderdocPathLower;
|
|
|
|
renderdocPath[idx] = 0;
|
|
|
|
wcscat_s(renderdocPath, L"\\Win32\\Development\\renderdoccmd.exe");
|
|
}
|
|
|
|
if(!devLocation)
|
|
{
|
|
devLocation = wcsstr(renderdocPathLower, L"\\x64\\release\\");
|
|
|
|
if(devLocation)
|
|
{
|
|
size_t idx = devLocation - renderdocPathLower;
|
|
|
|
renderdocPath[idx] = 0;
|
|
|
|
wcscat_s(renderdocPath, L"\\Win32\\Release\\renderdoccmd.exe");
|
|
}
|
|
}
|
|
|
|
if(!devLocation)
|
|
{
|
|
// look in a subfolder for x86.
|
|
|
|
// remove the filename from the path
|
|
wchar_t *slash = wcsrchr(renderdocPath, L'\\');
|
|
|
|
if(slash)
|
|
*slash = 0;
|
|
|
|
// append path
|
|
wcscat_s(renderdocPath, L"\\x86\\renderdoccmd.exe");
|
|
}
|
|
#else
|
|
// if it looks like we're in the development environment, look for the alternate bitness in the
|
|
// corresponding folder
|
|
const wchar_t *devLocation = wcsstr(renderdocPathLower, L"\\win32\\development\\");
|
|
if(devLocation)
|
|
{
|
|
size_t idx = devLocation - renderdocPathLower;
|
|
|
|
renderdocPath[idx] = 0;
|
|
|
|
wcscat_s(renderdocPath, L"\\x64\\Development\\renderdoccmd.exe");
|
|
}
|
|
|
|
if(!devLocation)
|
|
{
|
|
devLocation = wcsstr(renderdocPathLower, L"\\win32\\release\\");
|
|
|
|
if(devLocation)
|
|
{
|
|
size_t idx = devLocation - renderdocPathLower;
|
|
|
|
renderdocPath[idx] = 0;
|
|
|
|
wcscat_s(renderdocPath, L"\\x64\\Release\\renderdoccmd.exe");
|
|
}
|
|
}
|
|
|
|
if(!devLocation)
|
|
{
|
|
// look upwards on 32-bit to find the parent renderdoccmd.
|
|
wchar_t *slash = wcsrchr(renderdocPath, L'\\');
|
|
|
|
// remove the filename
|
|
if(slash)
|
|
*slash = 0;
|
|
|
|
// remove the \\x86
|
|
slash = wcsrchr(renderdocPath, L'\\');
|
|
|
|
if(slash)
|
|
*slash = 0;
|
|
|
|
// append path
|
|
wcscat_s(renderdocPath, L"\\renderdoccmd.exe");
|
|
}
|
|
#endif
|
|
|
|
PROCESS_INFORMATION pi;
|
|
STARTUPINFO si;
|
|
SECURITY_ATTRIBUTES pSec;
|
|
SECURITY_ATTRIBUTES tSec;
|
|
|
|
RDCEraseEl(pi);
|
|
RDCEraseEl(si);
|
|
RDCEraseEl(pSec);
|
|
RDCEraseEl(tSec);
|
|
|
|
// hide the console window
|
|
si.cb = sizeof(si);
|
|
si.dwFlags |= STARTF_USESHOWWINDOW;
|
|
si.wShowWindow = SW_HIDE;
|
|
|
|
pSec.nLength = sizeof(pSec);
|
|
tSec.nLength = sizeof(tSec);
|
|
|
|
// serialise to string with two chars per byte
|
|
rdcstr optstr = opts.EncodeAsString();
|
|
|
|
wchar_t *paramsAlloc = new wchar_t[2048];
|
|
|
|
rdcstr debugLogfile = RDCGETLOGFILE();
|
|
rdcwstr wdebugLogfile = StringFormat::UTF82Wide(debugLogfile);
|
|
|
|
_snwprintf_s(
|
|
paramsAlloc, 2047, 2047,
|
|
L"\"%ls\" capaltbit --pid=%d --capfile=\"%ls\" --debuglog=\"%ls\" --capopts=\"%hs\"",
|
|
renderdocPath, pid, wcapturefile.c_str(), wdebugLogfile.c_str(), optstr.c_str());
|
|
|
|
RDCDEBUG("params %ls", paramsAlloc);
|
|
|
|
paramsAlloc[2047] = 0;
|
|
|
|
wchar_t *commandLine = paramsAlloc;
|
|
|
|
std::wstring cmdWithEnv;
|
|
|
|
if(!env.empty())
|
|
{
|
|
cmdWithEnv = paramsAlloc;
|
|
|
|
for(const EnvironmentModification &e : env)
|
|
{
|
|
rdcstr name = e.name.trimmed();
|
|
rdcstr value = e.value;
|
|
|
|
if(name == "")
|
|
break;
|
|
|
|
cmdWithEnv += L" +env-";
|
|
switch(e.mod)
|
|
{
|
|
case EnvMod::Set: cmdWithEnv += L"replace"; break;
|
|
case EnvMod::Append: cmdWithEnv += L"append"; break;
|
|
case EnvMod::Prepend: cmdWithEnv += L"prepend"; break;
|
|
}
|
|
|
|
if(e.mod != EnvMod::Set)
|
|
{
|
|
switch(e.sep)
|
|
{
|
|
case EnvSep::Platform: cmdWithEnv += L"-platform"; break;
|
|
case EnvSep::SemiColon: cmdWithEnv += L"-semicolon"; break;
|
|
case EnvSep::Colon: cmdWithEnv += L"-colon"; break;
|
|
case EnvSep::NoSep: break;
|
|
}
|
|
}
|
|
|
|
cmdWithEnv += L" ";
|
|
|
|
// escape the parameters
|
|
for(size_t it = 0; it < name.size(); it++)
|
|
{
|
|
if(name[it] == '"')
|
|
{
|
|
name.insert(it, '\\');
|
|
it++;
|
|
}
|
|
}
|
|
|
|
for(size_t it = 0; it < value.size(); it++)
|
|
{
|
|
if(value[it] == '"')
|
|
{
|
|
value.insert(it, '\\');
|
|
it++;
|
|
}
|
|
}
|
|
|
|
if(name.back() == '\\')
|
|
name += "\\";
|
|
|
|
if(value.back() == '\\')
|
|
value += "\\";
|
|
|
|
cmdWithEnv += L"\"" + std::wstring(StringFormat::UTF82Wide(name).c_str()) + L"\" ";
|
|
cmdWithEnv += L"\"" + std::wstring(StringFormat::UTF82Wide(value).c_str()) + L"\" ";
|
|
}
|
|
|
|
commandLine = (wchar_t *)cmdWithEnv.c_str();
|
|
}
|
|
|
|
BOOL retValue = CreateProcessW(NULL, commandLine, &pSec, &tSec, false,
|
|
CREATE_NEW_CONSOLE | CREATE_SUSPENDED, NULL, NULL, &si, &pi);
|
|
|
|
SAFE_DELETE_ARRAY(paramsAlloc);
|
|
|
|
if(!retValue)
|
|
{
|
|
RDCERR(
|
|
"Can't spawn alternate bitness renderdoccmd - have you built 32-bit and 64-bit?\n"
|
|
"You need to build the matching bitness for the programs you want to capture.");
|
|
return {ReplayStatus::InternalError, 0};
|
|
}
|
|
|
|
ResumeThread(pi.hThread);
|
|
WaitForSingleObject(pi.hThread, INFINITE);
|
|
CloseHandle(pi.hThread);
|
|
|
|
DWORD exitCode = 0;
|
|
GetExitCodeProcess(pi.hProcess, &exitCode);
|
|
CloseHandle(pi.hProcess);
|
|
|
|
if(waitForExit)
|
|
WaitForSingleObject(hProcess, INFINITE);
|
|
|
|
CloseHandle(hProcess);
|
|
|
|
if(exitCode == 0)
|
|
return {ReplayStatus::UnknownError, 0};
|
|
if(exitCode < RenderDoc_FirstTargetControlPort)
|
|
return {(ReplayStatus)exitCode, 0};
|
|
|
|
return {ReplayStatus::Succeeded, (uint32_t)exitCode};
|
|
}
|
|
|
|
InjectDLL(hProcess, renderdocPath);
|
|
|
|
uintptr_t loc = FindRemoteDLL(pid, STRINGIZE(RDOC_DLL_FILE) ".dll");
|
|
|
|
rdcpair<ReplayStatus, uint32_t> result = {ReplayStatus::Succeeded, 0};
|
|
|
|
if(loc == 0)
|
|
{
|
|
RDCERR("Can't locate " STRINGIZE(RDOC_DLL_FILE) ".dll in remote PID %d", pid);
|
|
result.first = ReplayStatus::InjectionFailed;
|
|
}
|
|
else
|
|
{
|
|
// safe to cast away the const as we know these functions don't modify the parameters
|
|
|
|
if(capturefile != NULL)
|
|
InjectFunctionCall(hProcess, loc, "INTERNAL_SetCaptureFile", (void *)capturefile,
|
|
strlen(capturefile) + 1);
|
|
|
|
rdcstr debugLogfile = RDCGETLOGFILE();
|
|
|
|
InjectFunctionCall(hProcess, loc, "RENDERDOC_SetDebugLogFile", (void *)debugLogfile.c_str(),
|
|
debugLogfile.size() + 1);
|
|
|
|
InjectFunctionCall(hProcess, loc, "INTERNAL_SetCaptureOptions", (CaptureOptions *)&opts,
|
|
sizeof(CaptureOptions));
|
|
|
|
InjectFunctionCall(hProcess, loc, "INTERNAL_GetTargetControlIdent", &result.second,
|
|
sizeof(result.second));
|
|
|
|
if(!env.empty())
|
|
{
|
|
for(const EnvironmentModification &e : env)
|
|
{
|
|
rdcstr name = e.name.trimmed();
|
|
rdcstr value = e.value;
|
|
EnvMod mod = e.mod;
|
|
EnvSep sep = e.sep;
|
|
|
|
if(name == "")
|
|
break;
|
|
|
|
InjectFunctionCall(hProcess, loc, "INTERNAL_EnvModName", (void *)name.c_str(),
|
|
name.size() + 1);
|
|
InjectFunctionCall(hProcess, loc, "INTERNAL_EnvModValue", (void *)value.c_str(),
|
|
value.size() + 1);
|
|
InjectFunctionCall(hProcess, loc, "INTERNAL_EnvSep", &sep, sizeof(sep));
|
|
InjectFunctionCall(hProcess, loc, "INTERNAL_EnvMod", &mod, sizeof(mod));
|
|
}
|
|
|
|
// parameter is unused
|
|
void *dummy = NULL;
|
|
InjectFunctionCall(hProcess, loc, "INTERNAL_ApplyEnvMods", &dummy, sizeof(dummy));
|
|
}
|
|
}
|
|
|
|
if(waitForExit)
|
|
WaitForSingleObject(hProcess, INFINITE);
|
|
|
|
CloseHandle(hProcess);
|
|
|
|
return result;
|
|
}
|
|
|
|
uint32_t Process::LaunchProcess(const char *app, const char *workingDir, const char *cmdLine,
|
|
bool internal, ProcessResult *result)
|
|
{
|
|
HANDLE hChildStdOutput_Rd = NULL, hChildStdError_Rd = NULL;
|
|
|
|
PROCESS_INFORMATION pi =
|
|
RunProcess(app, workingDir, cmdLine, {}, internal, result ? &hChildStdOutput_Rd : NULL,
|
|
result ? &hChildStdError_Rd : NULL);
|
|
|
|
if(pi.dwProcessId == 0)
|
|
{
|
|
if(!internal)
|
|
RDCWARN("Couldn't launch process '%s'", app);
|
|
return 0;
|
|
}
|
|
|
|
if(!internal)
|
|
RDCLOG("Launched process '%s' with '%s'", app, cmdLine);
|
|
|
|
ResumeThread(pi.hThread);
|
|
|
|
if(result)
|
|
{
|
|
result->strStdout = "";
|
|
result->strStderror = "";
|
|
|
|
char chBuf[4096];
|
|
DWORD dwOutputRead, dwErrorRead;
|
|
BOOL success = FALSE;
|
|
rdcstr s;
|
|
for(;;)
|
|
{
|
|
success = ReadFile(hChildStdOutput_Rd, chBuf, sizeof(chBuf), &dwOutputRead, NULL);
|
|
s = rdcstr(chBuf, dwOutputRead);
|
|
result->strStdout += s;
|
|
|
|
if(!success && !dwOutputRead)
|
|
break;
|
|
}
|
|
|
|
for(;;)
|
|
{
|
|
success = ReadFile(hChildStdError_Rd, chBuf, sizeof(chBuf), &dwErrorRead, NULL);
|
|
s = rdcstr(chBuf, dwErrorRead);
|
|
result->strStderror += s;
|
|
|
|
if(!success && !dwErrorRead)
|
|
break;
|
|
}
|
|
|
|
CloseHandle(hChildStdOutput_Rd);
|
|
CloseHandle(hChildStdError_Rd);
|
|
|
|
WaitForSingleObject(pi.hProcess, INFINITE);
|
|
GetExitCodeProcess(pi.hProcess, (LPDWORD)&result->retCode);
|
|
}
|
|
|
|
CloseHandle(pi.hThread);
|
|
CloseHandle(pi.hProcess);
|
|
|
|
return pi.dwProcessId;
|
|
}
|
|
|
|
uint32_t Process::LaunchScript(const char *script, const char *workingDir, const char *argList,
|
|
bool internal, ProcessResult *result)
|
|
{
|
|
// Change parameters to invoke command interpreter
|
|
rdcstr args = "/C " + rdcstr(script) + " " + rdcstr(argList);
|
|
|
|
return LaunchProcess("cmd.exe", workingDir, args.c_str(), internal, result);
|
|
}
|
|
|
|
rdcpair<ReplayStatus, uint32_t> Process::LaunchAndInjectIntoProcess(
|
|
const char *app, const char *workingDir, const char *cmdLine,
|
|
const rdcarray<EnvironmentModification> &env, const char *capturefile,
|
|
const CaptureOptions &opts, bool waitForExit)
|
|
{
|
|
void *func =
|
|
GetProcAddress(GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll"), "INTERNAL_SetCaptureFile");
|
|
|
|
if(func == NULL)
|
|
{
|
|
RDCERR("Can't find required export function in " STRINGIZE(
|
|
RDOC_DLL_FILE) ".dll - corrupted/missing file?");
|
|
return {ReplayStatus::InternalError, 0};
|
|
}
|
|
|
|
PROCESS_INFORMATION pi = RunProcess(app, workingDir, cmdLine, env, false, NULL, NULL);
|
|
|
|
if(pi.dwProcessId == 0)
|
|
return {ReplayStatus::InjectionFailed, 0};
|
|
|
|
rdcpair<ReplayStatus, uint32_t> ret =
|
|
InjectIntoProcess(pi.dwProcessId, {}, capturefile, opts, false);
|
|
|
|
CloseHandle(pi.hProcess);
|
|
ResumeThread(pi.hThread);
|
|
ResumeThread(pi.hThread);
|
|
|
|
if(ret.second == 0 || ret.first != ReplayStatus::Succeeded)
|
|
{
|
|
CloseHandle(pi.hThread);
|
|
return ret;
|
|
}
|
|
|
|
if(waitForExit)
|
|
WaitForSingleObject(pi.hThread, INFINITE);
|
|
|
|
CloseHandle(pi.hThread);
|
|
|
|
return ret;
|
|
}
|
|
|
|
bool Process::CanGlobalHook()
|
|
{
|
|
// all we need is admin rights and it's the caller's responsibility to ensure that.
|
|
return true;
|
|
}
|
|
|
|
// to simplify the below code, rather than splitting by 32-bit/64-bit we split by native and Wow32.
|
|
// This means that for 32-bit code (whether it's on 32-bit OS or not) we just have native, and the
|
|
// Wow32 stuff is empty/unused. For 64-bit we use both. Thus the native registry key is always the
|
|
// same path regardless of the bitness we're running as and we don't have to move things around or
|
|
// have conditionals all over
|
|
|
|
struct GlobalHookData
|
|
{
|
|
struct
|
|
{
|
|
HANDLE pipe = NULL;
|
|
DWORD appinitEnabled = 0;
|
|
rdcwstr appinitDLLs;
|
|
} dataNative, dataWow32;
|
|
|
|
volatile int32_t finished = 0;
|
|
Threading::ThreadHandle pipeThread = 0;
|
|
};
|
|
|
|
// utility function to close the registry keys, print an error, and quit
|
|
static bool HandleRegError(HKEY keyNative, HKEY keyWow32, LSTATUS ret, const char *msg)
|
|
{
|
|
if(keyNative)
|
|
RegCloseKey(keyNative);
|
|
|
|
if(keyWow32)
|
|
RegCloseKey(keyWow32);
|
|
|
|
RDCERR("Error with AppInit registry keys - %s (%d)", msg, ret);
|
|
return false;
|
|
}
|
|
|
|
#define REG_CHECK(msg) \
|
|
if(ret != ERROR_SUCCESS) \
|
|
{ \
|
|
return HandleRegError(keyNative, keyWow32, ret, msg); \
|
|
}
|
|
|
|
// function to backup the previous settings for AppInit, then enable it and write our own paths.
|
|
bool BackupAndChangeRegistry(GlobalHookData &hookdata, const rdcstr &shimpathWow32,
|
|
const rdcstr &shimpathNative)
|
|
{
|
|
HKEY keyNative = NULL;
|
|
HKEY keyWow32 = NULL;
|
|
|
|
// open the native key
|
|
LSTATUS ret = RegCreateKeyExA(HKEY_LOCAL_MACHINE,
|
|
"SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows", 0, NULL,
|
|
0, KEY_READ | KEY_WRITE, NULL, &keyNative, NULL);
|
|
|
|
REG_CHECK("Could not open AppInit key");
|
|
|
|
// if we are doing Wow32, open that key as well
|
|
if(!shimpathWow32.empty())
|
|
{
|
|
ret = RegCreateKeyExA(HKEY_LOCAL_MACHINE,
|
|
"SOFTWARE\\Wow6432Node\\Microsoft\\Windows NT\\CurrentVersion\\Windows",
|
|
0, NULL, 0, KEY_READ | KEY_WRITE, NULL, &keyWow32, NULL);
|
|
|
|
REG_CHECK("Could not open AppInit key");
|
|
}
|
|
|
|
const DWORD one = 1;
|
|
|
|
// fetch the previous data for LoadAppInit_DLLs and AppInit_DLLs
|
|
DWORD sz = 4;
|
|
ret = RegGetValueA(keyNative, NULL, "LoadAppInit_DLLs", RRF_RT_REG_DWORD, NULL,
|
|
(void *)&hookdata.dataNative.appinitEnabled, &sz);
|
|
REG_CHECK("Could not fetch LoadAppInit_DLLs");
|
|
|
|
sz = 0;
|
|
ret = RegGetValueW(keyNative, NULL, L"AppInit_DLLs", RRF_RT_ANY, NULL, NULL, &sz);
|
|
if(ret == ERROR_MORE_DATA || ret == ERROR_SUCCESS)
|
|
{
|
|
hookdata.dataNative.appinitDLLs = rdcwstr(sz / sizeof(wchar_t));
|
|
ret = RegGetValueW(keyNative, NULL, L"AppInit_DLLs", RRF_RT_ANY, NULL,
|
|
hookdata.dataNative.appinitDLLs.data(), &sz);
|
|
}
|
|
REG_CHECK("Could not fetch AppInit_DLLs");
|
|
|
|
// set DWORD:1 for LoadAppInit_DLLs and convert our path to a short path then set it
|
|
ret = RegSetValueExA(keyNative, "LoadAppInit_DLLs", 0, REG_DWORD, (const BYTE *)&one, sizeof(one));
|
|
REG_CHECK("Could not set LoadAppInit_DLLs");
|
|
|
|
rdcwstr shortpath(shimpathNative.size());
|
|
GetShortPathNameW(StringFormat::UTF82Wide(shimpathNative).c_str(), shortpath.data(),
|
|
(DWORD)shortpath.length());
|
|
|
|
ret = RegSetValueExW(keyNative, L"AppInit_DLLs", 0, REG_SZ, (const BYTE *)shortpath.data(),
|
|
DWORD(shortpath.length() * sizeof(wchar_t)));
|
|
REG_CHECK("Could not set AppInit_DLLs");
|
|
|
|
// if we're doing Wow32, repeat the process for those keys
|
|
if(keyWow32)
|
|
{
|
|
sz = 4;
|
|
ret = RegGetValueA(keyWow32, NULL, "LoadAppInit_DLLs", RRF_RT_REG_DWORD, NULL,
|
|
(void *)&hookdata.dataWow32.appinitEnabled, &sz);
|
|
REG_CHECK("Could not fetch LoadAppInit_DLLs");
|
|
|
|
sz = 0;
|
|
ret = RegGetValueW(keyWow32, NULL, L"AppInit_DLLs", RRF_RT_ANY, NULL, NULL, &sz);
|
|
if(ret == ERROR_MORE_DATA || ret == ERROR_SUCCESS)
|
|
{
|
|
hookdata.dataWow32.appinitDLLs = rdcwstr(sz / sizeof(wchar_t));
|
|
ret = RegGetValueW(keyWow32, NULL, L"AppInit_DLLs", RRF_RT_ANY, NULL,
|
|
hookdata.dataWow32.appinitDLLs.data(), &sz);
|
|
}
|
|
REG_CHECK("Could not fetch AppInit_DLLs");
|
|
|
|
ret = RegSetValueExA(keyWow32, "LoadAppInit_DLLs", 0, REG_DWORD, (const BYTE *)&one, sizeof(one));
|
|
REG_CHECK("Could not set LoadAppInit_DLLs");
|
|
|
|
shortpath = rdcwstr(shimpathWow32.size());
|
|
GetShortPathNameW(StringFormat::UTF82Wide(shimpathWow32).c_str(), shortpath.data(),
|
|
(DWORD)shortpath.length());
|
|
|
|
ret = RegSetValueExW(keyWow32, L"AppInit_DLLs", 0, REG_SZ, (const BYTE *)shortpath.data(),
|
|
DWORD(shortpath.length() * sizeof(wchar_t)));
|
|
REG_CHECK("Could not set AppInit_DLLs");
|
|
}
|
|
|
|
std::wstring backup;
|
|
|
|
// write a .reg file that contains the previous settings, so that if all else fails the user can
|
|
// manually insert it back into the registry to restore everything.
|
|
backup += L"Windows Registry Editor Version 5.00\n";
|
|
backup += L"\n";
|
|
backup += L"[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows]\n";
|
|
backup += L"\"LoadAppInit_DLLs\"=dword:0000000";
|
|
backup += (hookdata.dataNative.appinitEnabled ? L"1\n" : L"0\n");
|
|
backup += L"\"AppInit_DLLs\"=\"";
|
|
// we append with the C string so we don't add trailing NULLs into the text.
|
|
backup += hookdata.dataNative.appinitDLLs.c_str();
|
|
backup += L"\"\n";
|
|
if(keyWow32)
|
|
{
|
|
backup += L"\n";
|
|
backup +=
|
|
L"[HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\Microsoft\\"
|
|
L"Windows NT\\CurrentVersion\\Windows]\n";
|
|
backup += L"\"LoadAppInit_DLLs\"=dword:0000000";
|
|
backup += (hookdata.dataWow32.appinitEnabled ? L"1\n" : L"0\n");
|
|
backup += L"\"AppInit_DLLs\"=\"";
|
|
backup += hookdata.dataWow32.appinitDLLs.c_str();
|
|
backup += L"\"\n";
|
|
}
|
|
|
|
if(keyNative)
|
|
RegCloseKey(keyNative);
|
|
|
|
if(keyWow32)
|
|
RegCloseKey(keyWow32);
|
|
|
|
keyNative = keyWow32 = NULL;
|
|
|
|
// write it to disk but don't fail if we can't, just print it to the log and keep going.
|
|
wchar_t reg_backup[MAX_PATH];
|
|
GetTempPathW(MAX_PATH, reg_backup);
|
|
wcscat_s(reg_backup, L"RenderDoc_RestoreGlobalHook.reg");
|
|
|
|
FILE *f = NULL;
|
|
_wfopen_s(&f, reg_backup, L"w");
|
|
if(f)
|
|
{
|
|
fputws(backup.c_str(), f);
|
|
fclose(f);
|
|
}
|
|
else
|
|
{
|
|
RDCERR("Error opening registry backup file %ls", reg_backup);
|
|
RDCERR("Backup registry data is:\n\n%ls\n\n", backup.c_str());
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// switch error-handling to print-and-continue, as we can't really do anything about it at this
|
|
// point and we want to continue restoring in case only one thing failed.
|
|
#undef REG_CHECK
|
|
#define REG_CHECK(msg) \
|
|
if(ret != ERROR_SUCCESS) \
|
|
{ \
|
|
HandleRegError(keyNative, keyWow32, ret, "Could not open AppInit key"); \
|
|
}
|
|
|
|
void RestoreRegistry(const GlobalHookData &hookdata)
|
|
{
|
|
HKEY keyNative = NULL;
|
|
HKEY keyWow32 = NULL;
|
|
LSTATUS ret = RegCreateKeyExA(HKEY_LOCAL_MACHINE,
|
|
"SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows", 0, NULL,
|
|
0, KEY_READ | KEY_WRITE, NULL, &keyNative, NULL);
|
|
|
|
REG_CHECK("Could not open AppInit key");
|
|
|
|
#if ENABLED(RDOC_X64)
|
|
ret = RegCreateKeyExA(HKEY_LOCAL_MACHINE,
|
|
"SOFTWARE\\Wow6432Node\\Microsoft\\Windows NT\\CurrentVersion\\Windows", 0,
|
|
NULL, 0, KEY_READ | KEY_WRITE, NULL, &keyWow32, NULL);
|
|
|
|
REG_CHECK("Could not open AppInit key");
|
|
#endif
|
|
|
|
// set the native values back to where they were
|
|
ret = RegSetValueExA(keyNative, "LoadAppInit_DLLs", 0, REG_DWORD,
|
|
(const BYTE *)&hookdata.dataNative.appinitEnabled,
|
|
sizeof(hookdata.dataNative.appinitEnabled));
|
|
REG_CHECK("Could not set LoadAppInit_DLLs");
|
|
|
|
ret = RegSetValueExW(keyNative, L"AppInit_DLLs", 0, REG_SZ,
|
|
(const BYTE *)hookdata.dataNative.appinitDLLs.c_str(),
|
|
DWORD(hookdata.dataNative.appinitDLLs.length() * sizeof(wchar_t)));
|
|
REG_CHECK("Could not set AppInit_DLLs");
|
|
|
|
// if we opened it, restore the Wow32 values as well
|
|
if(keyWow32)
|
|
{
|
|
ret = RegSetValueExA(keyWow32, "LoadAppInit_DLLs", 0, REG_DWORD,
|
|
(const BYTE *)&hookdata.dataWow32.appinitEnabled,
|
|
sizeof(hookdata.dataWow32.appinitEnabled));
|
|
REG_CHECK("Could not set LoadAppInit_DLLs");
|
|
|
|
ret = RegSetValueExW(keyWow32, L"AppInit_DLLs", 0, REG_SZ,
|
|
(const BYTE *)hookdata.dataWow32.appinitDLLs.c_str(),
|
|
DWORD(hookdata.dataWow32.appinitDLLs.length() * sizeof(wchar_t)));
|
|
REG_CHECK("Could not set AppInit_DLLs");
|
|
}
|
|
}
|
|
|
|
static GlobalHookData *globalHook = NULL;
|
|
|
|
// a thread we run in the background just to keep the pipes open and wait until we're ready to stop
|
|
// the global hook.
|
|
static void GlobalHookThread()
|
|
{
|
|
// keep looping doing an atomic compare-exchange to check that finished is still 0
|
|
while(Atomic::CmpExch32(&globalHook->finished, 0, 0) == 0)
|
|
{
|
|
// wake every quarter of a second to test again
|
|
Threading::Sleep(250);
|
|
}
|
|
|
|
char exitData[32] = "exit";
|
|
|
|
// write some data into the pipe and close it. The data is (currently) unimportant, just that it
|
|
// causes the blocking read on the other end to succeed and close the program.
|
|
DWORD dummy = 0;
|
|
if(globalHook->dataNative.pipe)
|
|
{
|
|
WriteFile(globalHook->dataNative.pipe, exitData, (DWORD)sizeof(exitData), &dummy, NULL);
|
|
CloseHandle(globalHook->dataNative.pipe);
|
|
}
|
|
|
|
if(globalHook->dataWow32.pipe)
|
|
{
|
|
WriteFile(globalHook->dataWow32.pipe, exitData, (DWORD)sizeof(exitData), &dummy, NULL);
|
|
CloseHandle(globalHook->dataWow32.pipe);
|
|
}
|
|
}
|
|
|
|
bool Process::StartGlobalHook(const char *pathmatch, const char *capturefile,
|
|
const CaptureOptions &opts)
|
|
{
|
|
if(pathmatch == NULL)
|
|
return false;
|
|
|
|
rdcstr renderdocPath;
|
|
FileIO::GetLibraryFilename(renderdocPath);
|
|
|
|
renderdocPath = get_basename(renderdocPath);
|
|
|
|
// the native renderdoccmd.exe is always next to the dll. Wow32 will be somewhere else
|
|
rdcstr cmdpathNative = renderdocPath + "\\renderdoccmd.exe";
|
|
rdcstr cmdpathWow32;
|
|
|
|
rdcstr shimpathNative = renderdocPath;
|
|
rdcstr shimpathWow32;
|
|
|
|
#if ENABLED(RDOC_X64)
|
|
|
|
// native shim is just renderdocshim64.dll
|
|
shimpathNative = renderdocPath + "\\renderdocshim64.dll";
|
|
|
|
// if it looks like we're in the development environment, look for the alternate bitness in the
|
|
// corresponding folder
|
|
int devLocation = renderdocPath.find("\\x64\\Development\\");
|
|
if(devLocation >= 0)
|
|
{
|
|
renderdocPath[devLocation] = 0;
|
|
|
|
shimpathWow32 = renderdocPath + "\\Win32\\Development\\renderdocshim32.dll";
|
|
cmdpathWow32 = renderdocPath + "\\Win32\\Development\\renderdoccmd.exe";
|
|
}
|
|
else
|
|
{
|
|
devLocation = renderdocPath.find("\\x64\\Release\\");
|
|
|
|
if(devLocation >= 0)
|
|
{
|
|
renderdocPath[devLocation] = 0;
|
|
|
|
shimpathWow32 = renderdocPath + "\\Win32\\Release\\renderdocshim32.dll";
|
|
cmdpathWow32 = renderdocPath + "\\Win32\\Release\\renderdoccmd.exe";
|
|
}
|
|
}
|
|
|
|
// if we're not in the dev environment, assume it's under a x86\ subfolder
|
|
if(devLocation < 0)
|
|
{
|
|
shimpathWow32 = renderdocPath + "\\x86\\renderdocshim32.dll";
|
|
cmdpathWow32 = renderdocPath + "\\x86\\renderdoccmd.exe";
|
|
}
|
|
|
|
#else
|
|
|
|
// nothing fancy to do here for 32-bit, just point the shim next to our dll.
|
|
shimpathNative = renderdocPath + "\\renderdocshim32.dll";
|
|
|
|
#endif
|
|
|
|
GlobalHookData hookdata;
|
|
|
|
// try to backup and change the registry settings to start loading our shim dlls. If that fails,
|
|
// we bail out immediately
|
|
bool success = BackupAndChangeRegistry(hookdata, shimpathWow32, shimpathNative);
|
|
if(!success)
|
|
return false;
|
|
|
|
PROCESS_INFORMATION pi = {0};
|
|
STARTUPINFO si = {0};
|
|
SECURITY_ATTRIBUTES pSec = {0};
|
|
SECURITY_ATTRIBUTES tSec = {0};
|
|
pSec.nLength = sizeof(pSec);
|
|
tSec.nLength = sizeof(tSec);
|
|
|
|
si.cb = sizeof(si);
|
|
|
|
// serialise to string with two chars per byte
|
|
rdcstr optstr = opts.EncodeAsString();
|
|
rdcstr debugLogfile = RDCGETLOGFILE();
|
|
|
|
rdcstr params = StringFormat::Fmt(
|
|
"\"%s\" globalhook --match \"%s\" --capfile \"%s\" --debuglog \"%s\" --capopts \"%s\"",
|
|
cmdpathNative.c_str(), pathmatch, capturefile ? capturefile : "", debugLogfile.c_str(),
|
|
optstr.c_str());
|
|
|
|
rdcwstr paramsAlloc = StringFormat::UTF82Wide(params);
|
|
|
|
// we'll be setting stdin
|
|
si.dwFlags = STARTF_USESHOWWINDOW | STARTF_USESTDHANDLES;
|
|
|
|
// hide the console window
|
|
si.wShowWindow = SW_HIDE;
|
|
|
|
// this is the end of the pipe that the child will inherit and use as stdin
|
|
HANDLE childEnd = NULL;
|
|
|
|
// create a pipe with the writing end for us, and the reading end as the child process's stdin
|
|
{
|
|
SECURITY_ATTRIBUTES pipeSec;
|
|
pipeSec.nLength = sizeof(SECURITY_ATTRIBUTES);
|
|
pipeSec.bInheritHandle = TRUE;
|
|
pipeSec.lpSecurityDescriptor = NULL;
|
|
|
|
BOOL res;
|
|
res = CreatePipe(&childEnd, &hookdata.dataNative.pipe, &pipeSec, 0);
|
|
|
|
if(!res)
|
|
{
|
|
RDCERR("Could not create 32-bit stdin pipe");
|
|
RestoreRegistry(hookdata);
|
|
return false;
|
|
}
|
|
|
|
// we don't want the child process to inherit our end
|
|
res = SetHandleInformation(hookdata.dataNative.pipe, HANDLE_FLAG_INHERIT, 0);
|
|
|
|
if(!res)
|
|
{
|
|
RDCERR("Could not make 32-bit stdin pipe inheritable");
|
|
RestoreRegistry(hookdata);
|
|
return false;
|
|
}
|
|
|
|
si.hStdInput = childEnd;
|
|
}
|
|
|
|
// launch the process
|
|
BOOL retValue = CreateProcessW(NULL, ¶msAlloc[0], &pSec, &tSec, true, CREATE_NEW_CONSOLE,
|
|
NULL, NULL, &si, &pi);
|
|
|
|
// we don't need this end anymore, the child has it
|
|
CloseHandle(childEnd);
|
|
|
|
if(retValue == FALSE)
|
|
{
|
|
RDCERR("Can't launch 64-bit renderdoccmd from '%ls'", cmdpathNative.c_str());
|
|
CloseHandle(hookdata.dataNative.pipe);
|
|
RestoreRegistry(hookdata);
|
|
return false;
|
|
}
|
|
|
|
CloseHandle(pi.hThread);
|
|
CloseHandle(pi.hProcess);
|
|
|
|
RDCEraseEl(pi);
|
|
|
|
// repeat the process for the Wow32 renderdoccmd
|
|
#if ENABLED(RDOC_X64)
|
|
params = StringFormat::Fmt(
|
|
"\"%s\" globalhook --match \"%s\" --capfile \"%s\" --debuglog \"%s\" --capopts \"%s\"",
|
|
cmdpathWow32.c_str(), pathmatch, capturefile ? capturefile : "", debugLogfile.c_str(),
|
|
optstr.c_str());
|
|
|
|
paramsAlloc = StringFormat::UTF82Wide(params);
|
|
|
|
{
|
|
SECURITY_ATTRIBUTES pipeSec;
|
|
pipeSec.nLength = sizeof(SECURITY_ATTRIBUTES);
|
|
pipeSec.bInheritHandle = TRUE;
|
|
pipeSec.lpSecurityDescriptor = NULL;
|
|
|
|
BOOL res;
|
|
res = CreatePipe(&childEnd, &hookdata.dataWow32.pipe, &pipeSec, 0);
|
|
|
|
if(!res)
|
|
{
|
|
RDCERR("Could not create 64-bit stdin pipe");
|
|
RestoreRegistry(hookdata);
|
|
return false;
|
|
}
|
|
|
|
res = SetHandleInformation(hookdata.dataWow32.pipe, HANDLE_FLAG_INHERIT, 0);
|
|
|
|
if(!res)
|
|
{
|
|
RDCERR("Could not make 64-bit stdin pipe inheritable");
|
|
RestoreRegistry(hookdata);
|
|
return false;
|
|
}
|
|
|
|
si.hStdInput = childEnd;
|
|
}
|
|
|
|
retValue = CreateProcessW(NULL, ¶msAlloc[0], &pSec, &tSec, true, CREATE_NEW_CONSOLE, NULL,
|
|
NULL, &si, &pi);
|
|
|
|
// we don't need this end anymore
|
|
CloseHandle(childEnd);
|
|
|
|
if(retValue == FALSE)
|
|
{
|
|
RDCERR("Can't launch 32-bit renderdoccmd from '%ls'", cmdpathWow32.c_str());
|
|
CloseHandle(hookdata.dataNative.pipe);
|
|
CloseHandle(hookdata.dataWow32.pipe);
|
|
RestoreRegistry(hookdata);
|
|
return false;
|
|
}
|
|
|
|
CloseHandle(pi.hThread);
|
|
CloseHandle(pi.hProcess);
|
|
#endif
|
|
|
|
// set static global pointer with our data, and launch the thread
|
|
globalHook = new GlobalHookData;
|
|
*globalHook = hookdata;
|
|
|
|
globalHook->pipeThread = Threading::CreateThread(&GlobalHookThread);
|
|
|
|
return true;
|
|
}
|
|
|
|
bool Process::IsGlobalHookActive()
|
|
{
|
|
return globalHook != NULL;
|
|
}
|
|
void Process::StopGlobalHook()
|
|
{
|
|
if(!globalHook)
|
|
return;
|
|
|
|
// set the finished flag and join to the thread so it closes the pipes (and so the child
|
|
// processes)
|
|
Atomic::Inc32(&globalHook->finished);
|
|
|
|
Threading::JoinThread(globalHook->pipeThread);
|
|
Threading::CloseThread(globalHook->pipeThread);
|
|
|
|
// restore the registry settings from before we started
|
|
RestoreRegistry(*globalHook);
|
|
|
|
delete globalHook;
|
|
globalHook = NULL;
|
|
}
|
|
|
|
bool Process::IsModuleLoaded(const char *module)
|
|
{
|
|
return GetModuleHandleA(module) != NULL;
|
|
}
|
|
|
|
void *Process::LoadModule(const char *module)
|
|
{
|
|
HMODULE mod = GetModuleHandleA(module);
|
|
if(mod != NULL)
|
|
return mod;
|
|
|
|
return LoadLibraryA(module);
|
|
}
|
|
|
|
void *Process::GetFunctionAddress(void *module, const char *function)
|
|
{
|
|
if(module == NULL)
|
|
return NULL;
|
|
|
|
return (void *)GetProcAddress((HMODULE)module, function);
|
|
}
|
|
|
|
uint32_t Process::GetCurrentPID()
|
|
{
|
|
return (uint32_t)GetCurrentProcessId();
|
|
}
|
|
|
|
void Process::Shutdown()
|
|
{
|
|
// nothing to do
|
|
}
|