Ensure injected DLL doesn't run out of stack space

* GL hooks registration was using an obscene amount of stack because MSVC
  doesn't share stack for mutually exclusive locals (all the temporary
  FunctionHooks) so an executable that reduces the default thread stack size
  enough would fail to start.
* We also explicitly allocate 1MB of stack for injecting our DLL in future to
  avoid this issue.
This commit is contained in:
baldurk
2022-04-08 17:45:24 +01:00
parent 74178e8aeb
commit 3e5d912767
2 changed files with 21 additions and 9 deletions
+19 -7
View File
@@ -285,14 +285,26 @@ void GLHook::RegisterHooks()
LibraryHooks::RegisterLibraryHook(libraryName, &GLHooked);
#define RegisterFunc(func, name) \
LibraryHooks::RegisterFunctionHook( \
libraryName, \
FunctionHook(STRINGIZE(name), (void **)&GL.func, (void *)&CONCAT(func, _renderdoc_hooked)));
// MSVC compiles this function to use a huge amount of stack by initialising all the FunctionHook
// locals all at once. So we instead explicitly re-use the same hook (since it's going to be
// copied anyway, these are temporaries).
FunctionHook tmphook;
#define RegisterUnsupportedFunc(name) \
LibraryHooks::RegisterFunctionHook( \
libraryName, FunctionHook(STRINGIZE(name), NULL, (void *)&CONCAT(name, _renderdoc_hooked)));
#define RegisterFunc(func, name) \
{ \
tmphook.function = STRINGIZE(name); \
tmphook.orig = (void **)&GL.func; \
tmphook.hook = (void *)&CONCAT(func, _renderdoc_hooked); \
LibraryHooks::RegisterFunctionHook(libraryName, tmphook); \
}
#define RegisterUnsupportedFunc(name) \
{ \
tmphook.function = STRINGIZE(name); \
tmphook.orig = NULL; \
tmphook.hook = (void *)&CONCAT(name, _renderdoc_hooked); \
LibraryHooks::RegisterFunctionHook(libraryName, tmphook); \
}
ForEachSupported(RegisterFunc);
ForEachUnsupported(RegisterUnsupportedFunc);
+2 -2
View File
@@ -270,8 +270,8 @@ void InjectDLL(HANDLE hProcess, rdcwstr libName)
if(success)
{
HANDLE hThread = CreateRemoteThread(
hProcess, NULL, 0, (LPTHREAD_START_ROUTINE)GetProcAddress(kernel32, "LoadLibraryW"),
remoteMem, 0, NULL);
hProcess, NULL, 1024 * 1024U,
(LPTHREAD_START_ROUTINE)GetProcAddress(kernel32, "LoadLibraryW"), remoteMem, 0, NULL);
if(hThread)
{
WaitForSingleObject(hThread, INFINITE);