mirror of
https://github.com/baldurk/renderdoc.git
synced 2026-08-28 17:36:36 +00:00
Ensure injected DLL doesn't run out of stack space
* GL hooks registration was using an obscene amount of stack because MSVC doesn't share stack for mutually exclusive locals (all the temporary FunctionHooks) so an executable that reduces the default thread stack size enough would fail to start. * We also explicitly allocate 1MB of stack for injecting our DLL in future to avoid this issue.
This commit is contained in:
@@ -285,14 +285,26 @@ void GLHook::RegisterHooks()
|
||||
|
||||
LibraryHooks::RegisterLibraryHook(libraryName, &GLHooked);
|
||||
|
||||
#define RegisterFunc(func, name) \
|
||||
LibraryHooks::RegisterFunctionHook( \
|
||||
libraryName, \
|
||||
FunctionHook(STRINGIZE(name), (void **)&GL.func, (void *)&CONCAT(func, _renderdoc_hooked)));
|
||||
// MSVC compiles this function to use a huge amount of stack by initialising all the FunctionHook
|
||||
// locals all at once. So we instead explicitly re-use the same hook (since it's going to be
|
||||
// copied anyway, these are temporaries).
|
||||
FunctionHook tmphook;
|
||||
|
||||
#define RegisterUnsupportedFunc(name) \
|
||||
LibraryHooks::RegisterFunctionHook( \
|
||||
libraryName, FunctionHook(STRINGIZE(name), NULL, (void *)&CONCAT(name, _renderdoc_hooked)));
|
||||
#define RegisterFunc(func, name) \
|
||||
{ \
|
||||
tmphook.function = STRINGIZE(name); \
|
||||
tmphook.orig = (void **)&GL.func; \
|
||||
tmphook.hook = (void *)&CONCAT(func, _renderdoc_hooked); \
|
||||
LibraryHooks::RegisterFunctionHook(libraryName, tmphook); \
|
||||
}
|
||||
|
||||
#define RegisterUnsupportedFunc(name) \
|
||||
{ \
|
||||
tmphook.function = STRINGIZE(name); \
|
||||
tmphook.orig = NULL; \
|
||||
tmphook.hook = (void *)&CONCAT(name, _renderdoc_hooked); \
|
||||
LibraryHooks::RegisterFunctionHook(libraryName, tmphook); \
|
||||
}
|
||||
|
||||
ForEachSupported(RegisterFunc);
|
||||
ForEachUnsupported(RegisterUnsupportedFunc);
|
||||
|
||||
@@ -270,8 +270,8 @@ void InjectDLL(HANDLE hProcess, rdcwstr libName)
|
||||
if(success)
|
||||
{
|
||||
HANDLE hThread = CreateRemoteThread(
|
||||
hProcess, NULL, 0, (LPTHREAD_START_ROUTINE)GetProcAddress(kernel32, "LoadLibraryW"),
|
||||
remoteMem, 0, NULL);
|
||||
hProcess, NULL, 1024 * 1024U,
|
||||
(LPTHREAD_START_ROUTINE)GetProcAddress(kernel32, "LoadLibraryW"), remoteMem, 0, NULL);
|
||||
if(hThread)
|
||||
{
|
||||
WaitForSingleObject(hThread, INFINITE);
|
||||
|
||||
Reference in New Issue
Block a user