From 3e5d912767147d43f378a3b1460de7edeb7eb7fe Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Fri, 8 Apr 2022 11:50:35 +0100
Subject: [PATCH] Ensure injected DLL doesn't run out of stack space

* GL hooks registration was using an obscene amount of stack because MSVC
  doesn't share stack for mutually exclusive locals (all the temporary
  FunctionHooks) so an executable that reduces the default thread stack size
  enough would fail to start.
* We also explicitly allocate 1MB of stack for injecting our DLL in future to
  avoid this issue.
---
 renderdoc/driver/gl/gl_hooks.cpp     | 26 +++++++++++++++++++-------
 renderdoc/os/win32/win32_process.cpp |  4 ++--
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/renderdoc/driver/gl/gl_hooks.cpp b/renderdoc/driver/gl/gl_hooks.cpp
index 676626535..369f924be 100644
--- a/renderdoc/driver/gl/gl_hooks.cpp
+++ b/renderdoc/driver/gl/gl_hooks.cpp
@@ -285,14 +285,26 @@ void GLHook::RegisterHooks()
 
   LibraryHooks::RegisterLibraryHook(libraryName, &GLHooked);
 
-#define RegisterFunc(func, name)      \
-  LibraryHooks::RegisterFunctionHook( \
-      libraryName,                    \
-      FunctionHook(STRINGIZE(name), (void **)&GL.func, (void *)&CONCAT(func, _renderdoc_hooked)));
+  // MSVC compiles this function to use a huge amount of stack by initialising all the FunctionHook
+  // locals all at once. So we instead explicitly re-use the same hook (since it's going to be
+  // copied anyway, these are temporaries).
+  FunctionHook tmphook;
 
-#define RegisterUnsupportedFunc(name) \
-  LibraryHooks::RegisterFunctionHook( \
-      libraryName, FunctionHook(STRINGIZE(name), NULL, (void *)&CONCAT(name, _renderdoc_hooked)));
+#define RegisterFunc(func, name)                              \
+  {                                                           \
+    tmphook.function = STRINGIZE(name);                       \
+    tmphook.orig = (void **)&GL.func;                         \
+    tmphook.hook = (void *)&CONCAT(func, _renderdoc_hooked);  \
+    LibraryHooks::RegisterFunctionHook(libraryName, tmphook); \
+  }
+
+#define RegisterUnsupportedFunc(name)                         \
+  {                                                           \
+    tmphook.function = STRINGIZE(name);                       \
+    tmphook.orig = NULL;                                      \
+    tmphook.hook = (void *)&CONCAT(name, _renderdoc_hooked);  \
+    LibraryHooks::RegisterFunctionHook(libraryName, tmphook); \
+  }
 
   ForEachSupported(RegisterFunc);
   ForEachUnsupported(RegisterUnsupportedFunc);
diff --git a/renderdoc/os/win32/win32_process.cpp b/renderdoc/os/win32/win32_process.cpp
index dff7de0ba..d19933a62 100644
--- a/renderdoc/os/win32/win32_process.cpp
+++ b/renderdoc/os/win32/win32_process.cpp
@@ -270,8 +270,8 @@ void InjectDLL(HANDLE hProcess, rdcwstr libName)
     if(success)
     {
       HANDLE hThread = CreateRemoteThread(
-          hProcess, NULL, 0, (LPTHREAD_START_ROUTINE)GetProcAddress(kernel32, "LoadLibraryW"),
-          remoteMem, 0, NULL);
+          hProcess, NULL, 1024 * 1024U,
+          (LPTHREAD_START_ROUTINE)GetProcAddress(kernel32, "LoadLibraryW"), remoteMem, 0, NULL);
       if(hThread)
       {
         WaitForSingleObject(hThread, INFINITE);