Add IAT based hooking to allow hooking 64bit opengl32.dll

* Needs vigorous testing in as many situations as I can think of, not least
  32bit, 64bit, D3D11, D3D9, OpenGL, managed apps, different windows SDKs,
  delay loading dlls (not sure if these will work), statically linked libs,
  LoadLibrary + GetProcAddress
* Credit goes to jslutter for originally bringing it to my attention &
  suggesting it.
* References:
  http://msdn.microsoft.com/en-us/magazine/bb985992.aspx
  http://msdn.microsoft.com/en-us/magazine/cc301808.aspx
  http://www.codeproject.com/Articles/6265/Process-wide-API-spying-an-ultimate-hack

renderdoc/hooks/d3d11_hooks.cpp

@@ -39,12 +39,14 @@ public:
 	{
 		bool success = true;
 
+#if USE_MHOOK
 		// require dxgi.dll hooked as well for proper operation
 		if(GetModuleHandleA("dxgi.dll") == NULL)
 		{
 			RDCWARN("Failed to load dxgi.dll - not inserting D3D11 hooks.");
 			return false;
 		}
+#endif
 
 		// also require d3dcompiler_??.dll
 		if(GetD3DCompiler() == NULL)
@@ -58,6 +60,7 @@ public:
 
 		if(!success) return false;
 		
+#if USE_MHOOK
 		// FRAPS compatibility. Save out the first 16 bytes (arbitrary number) of the 'real' function code.
 		// this should be
 		// jmp D3D11CreateDeviceAndSwapChain_hook
@@ -68,6 +71,7 @@ public:
 		void *hooked_func_ptr = GetProcAddress(GetModuleHandleA("d3d11.dll"), "D3D11CreateDeviceAndSwapChain");
 		if(hooked_func_ptr == NULL) return false;
 		memcpy(CreateDeviceAndSwapChain_ident, hooked_func_ptr, 16);
+#endif
 
 		m_HasHooks = true;
 		m_EnabledHooks = true;
@@ -188,6 +192,7 @@ private:
 
 		PFN_D3D11_CREATE_DEVICE_AND_SWAP_CHAIN createFunc = (PFN_D3D11_CREATE_DEVICE_AND_SWAP_CHAIN)GetProcAddress(GetModuleHandleA("d3d11.dll"), "D3D11CreateDeviceAndSwapChain");
 		
+#if USE_MHOOK
 		if(createFunc)
 		{
 			byte ident[16];
@@ -211,6 +216,7 @@ private:
 			if(!memcmp(ident, CreateDeviceAndSwapChain_ident, 16) && m_HasHooks)
 				createFunc = CreateDeviceAndSwapChain();
 		}
+#endif
 
 		// shouldn't ever get here, we should either have it from procaddress or the trampoline, but let's be
 		// safe.

renderdoc/hooks/dxgi_hooks.cpp

@@ -40,8 +40,10 @@ public:
 	{
 		bool success = true;
 
+#if USE_MHOOK
 		// require d3d11.dll hooked as well for proper operation
 		if(GetModuleHandleA("d3d11.dll") == NULL) return false;
+#endif
 
 		success &= CreateDXGIFactory.Initialize("CreateDXGIFactory", DLL_NAME, CreateDXGIFactory_hook);
 		success &= CreateDXGIFactory1.Initialize("CreateDXGIFactory1", DLL_NAME, CreateDXGIFactory1_hook);

renderdoc/hooks/gl_win32_hooks.cpp

@@ -228,11 +228,13 @@ class OpenGLHook : LibraryHook
 			if(!m_EnabledHooks)
 				return false;
 			
+#if USE_MHOOK
 			if(GetModuleHandleA(DLL_NAME) == NULL)
 			{
 				RDCWARN("Failed to load %s - not inserting OpenGL hooks.", DLL_NAME);
 				return false;
 			}
+#endif
 
 			bool success = SetupHooks(GL);
 

renderdoc/hooks/hooks.h

@@ -35,36 +35,64 @@ using std::map;
 
 #if defined(WIN32)
 
-#include "3rdparty/mhook/mhook-lib/mhook.h"
+#define USE_MHOOK 0
+#define USE_IAT_HOOK 1
+
+#if USE_MHOOK
+	#include "3rdparty/mhook/mhook-lib/mhook.h"
+#elif USE_IAT_HOOK
+	#include "os/win32/win32_hook.h"
+#else
+	#error "No hook method enabled"
+#endif
 
 template<typename FuncType>
 class Hook
 {
 	public:
-		Hook() { trampoline = NULL; }
-		~Hook() { Mhook_Unhook(&trampoline); }
+		Hook()
+		{
+			orig_funcptr = NULL;
+		}
+		~Hook()
+		{
+#if USE_MHOOK
+			Mhook_Unhook(&orig_funcptr);
+#endif
+		}
 		
 		FuncType operator()()
 		{
-			return (FuncType)trampoline;
+			return (FuncType)orig_funcptr;
 		}
 
 		bool Initialize(const char *function, const char *module_name, void *destination_function_ptr)
 		{
-			trampoline = Process::GetFunctionAddress(module_name, function);
+			orig_funcptr = Process::GetFunctionAddress(module_name, function);
 
-			if(trampoline == NULL)
+#if USE_MHOOK
+			if(orig_funcptr == NULL)
 				return false;
-
-			return Mhook_SetHook(&trampoline, destination_function_ptr);
+			
+			return Mhook_SetHook(&orig_funcptr, destination_function_ptr);
+#elif USE_IAT_HOOK
+			return Win32_IAT_Hook(&orig_funcptr, module_name, function, destination_function_ptr);
+#else
+			#error "No hook method enabled"
+#endif
 		}
 
 	private:
-		void* trampoline;
+		void *orig_funcptr;
 };
 
-#define HOOKS_BEGIN() Mhook_SuspendOtherThreads()
-#define HOOKS_END() Mhook_ResumeOtherThreads()
+#if USE_MHOOK
+	#define HOOKS_BEGIN() Mhook_SuspendOtherThreads()
+	#define HOOKS_END() Mhook_ResumeOtherThreads()
+#else
+	#define HOOKS_BEGIN() Win32_IAT_BeginHooks()
+	#define HOOKS_END() Win32_IAT_EndHooks()
+#endif
 
 #elif defined(LINUX)
 

renderdoc/os/win32/win32_hook.cpp

@@ -0,0 +1,335 @@
+/******************************************************************************
+ * The MIT License (MIT)
+ * 
+ * Copyright (c) 2014 Crytek
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ ******************************************************************************/
+
+#include "os/os_specific.h"
+
+#include "common/string_utils.h"
+
+#include <windows.h>
+#include <tlhelp32.h> 
+
+#include <vector>
+#include <map>
+using std::vector;
+using std::map;
+
+struct FunctionHook
+{
+	FunctionHook(const char *f, void **o, void *d)
+		: function(f), origptr(o), hookptr(d), excludeModule(NULL)
+	{}
+	
+	bool operator <(const FunctionHook &h)
+	{
+		return function < h.function;
+	}
+
+	void ApplyHook(void **IATentry)
+	{
+		DWORD oldProtection = PAGE_EXECUTE;
+
+		BOOL success = TRUE;
+
+		success = VirtualProtect(IATentry, sizeof(void*), PAGE_READWRITE, &oldProtection);
+		if(!success)
+		{
+			RDCERR("Failed to make IAT entry writeable 0x%p", IATentry);
+			return;
+		}
+
+		if(origptr && *origptr == NULL && *IATentry != hookptr) *origptr = *IATentry;
+		
+		*IATentry = hookptr;
+
+		success = VirtualProtect(IATentry, sizeof(void*), oldProtection, &oldProtection);
+		if(!success)
+		{
+			RDCERR("Failed to restore IAT entry protection 0x%p", IATentry);
+			return;
+		}
+	}
+
+	string function;
+	void **origptr;
+	void *hookptr;
+	HMODULE excludeModule;
+};
+
+struct DllHookset
+{
+	DllHookset() : module(NULL) {}
+
+	HMODULE module;
+	vector<FunctionHook> FunctionHooks;
+};
+
+struct CachedHookData
+{
+	map<string, DllHookset> DllHooks;
+	HMODULE module;
+
+	void ApplyHooks(const char *modName, HMODULE module)
+	{
+		string name = strlower(string(modName));
+
+		// set module pointer if we are hooking exports from this module
+		for(auto it=DllHooks.begin(); it != DllHooks.end(); ++it)
+			if(it->first == name)
+				it->second.module = module;
+
+		byte *baseAddress = (byte *)module;
+
+		PIMAGE_DOS_HEADER dosheader = (PIMAGE_DOS_HEADER)baseAddress;
+
+		char *PE00 = (char *)(baseAddress + dosheader->e_lfanew);
+		PIMAGE_FILE_HEADER fileHeader = (PIMAGE_FILE_HEADER)(PE00+4);
+		PIMAGE_OPTIONAL_HEADER optHeader = (PIMAGE_OPTIONAL_HEADER)((BYTE *)fileHeader+sizeof(IMAGE_FILE_HEADER));
+
+		DWORD iatSize = optHeader->DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT].Size;
+		DWORD iatOffset = optHeader->DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT].VirtualAddress;
+
+		IMAGE_IMPORT_DESCRIPTOR *importDesc = (IMAGE_IMPORT_DESCRIPTOR *)(baseAddress + iatOffset);
+
+		while(iatOffset && importDesc->FirstThunk)
+		{
+			string dllName = strlower(string((const char *)(baseAddress + importDesc->Name)));
+
+			DllHookset *hookset = NULL;
+
+			for(auto it=DllHooks.begin(); it != DllHooks.end(); ++it)
+				if(it->first == dllName)
+					hookset = &it->second;
+
+			if(hookset)
+			{
+				IMAGE_THUNK_DATA *origFirst = (IMAGE_THUNK_DATA *)(baseAddress + importDesc->OriginalFirstThunk);
+				IMAGE_THUNK_DATA *first = (IMAGE_THUNK_DATA *)(baseAddress + importDesc->FirstThunk);
+
+				while(origFirst->u1.AddressOfData)
+				{
+#ifdef WIN64
+					if(origFirst->u1.AddressOfData & 0x8000000000000000)
+#else
+					if(origFirst->u1.AddressOfData & 0x80000000)
+#endif
+					{
+						// low bits of origFirst->u1.AddressOfData contain an ordinal
+						origFirst++;
+						first++;
+						continue;
+					}
+
+					IMAGE_IMPORT_BY_NAME *import = (IMAGE_IMPORT_BY_NAME *)(baseAddress + origFirst->u1.AddressOfData);
+					void **IATentry = (void **)&first->u1.AddressOfData;
+
+					FunctionHook search(import->Name, NULL, NULL);
+					auto found = std::lower_bound(hookset->FunctionHooks.begin(), hookset->FunctionHooks.end(), search);
+
+					if(found != hookset->FunctionHooks.end() && !(search < *found) && found->excludeModule != module)
+						found->ApplyHook(IATentry);
+
+					origFirst++;
+					first++;
+				}
+			}
+
+			importDesc++;
+		}
+	}
+};
+
+static CachedHookData *s_HookData = NULL;
+
+HMODULE WINAPI Hooked_LoadLibraryExA(LPCSTR lpLibFileName, HANDLE fileHandle, DWORD flags)
+{
+	// we can use the function naked, as when setting up the hook for LoadLibraryExA, our own module
+	// was excluded from IAT patching
+	HMODULE mod = LoadLibraryExA(lpLibFileName, fileHandle, flags);
+
+	if(mod)
+		s_HookData->ApplyHooks(lpLibFileName, mod);
+
+	return mod;
+}
+
+HMODULE WINAPI Hooked_LoadLibraryExW(LPCWSTR lpLibFileName, HANDLE fileHandle, DWORD flags)
+{
+	// we can use the function naked, as when setting up the hook for LoadLibraryExA, our own module
+	// was excluded from IAT patching
+	HMODULE mod = LoadLibraryExW(lpLibFileName, fileHandle, flags);
+
+	if(mod)
+		s_HookData->ApplyHooks(narrow(wstring(lpLibFileName)).c_str(), mod);
+
+	return mod;
+}
+
+HMODULE WINAPI Hooked_LoadLibraryA(LPCSTR lpLibFileName)
+{
+	return Hooked_LoadLibraryExA(lpLibFileName, NULL, 0);
+}
+
+HMODULE WINAPI Hooked_LoadLibraryW(LPCWSTR lpLibFileName)
+{
+	return Hooked_LoadLibraryExW(lpLibFileName, NULL, 0);
+}
+
+static bool OrdinalAsString(void *func)
+{
+	return uint64_t(func) <= 0xffff;
+}
+
+FARPROC WINAPI Hooked_GetProcAddress(HMODULE mod, LPCSTR func)
+{
+	if(mod == s_HookData->module || mod == NULL || func == NULL || OrdinalAsString((void *)func))
+		return GetProcAddress(mod, func);
+	
+	for(auto it=s_HookData->DllHooks.begin(); it != s_HookData->DllHooks.end(); ++it)
+	{
+		if(mod == it->second.module)
+		{
+			FunctionHook search(func, NULL, NULL);
+
+			for(size_t i=0; i < it->second.FunctionHooks.size(); i++)
+			{
+				auto found = std::lower_bound(it->second.FunctionHooks.begin(), it->second.FunctionHooks.end(), search);
+				if(found != it->second.FunctionHooks.end() && !(search < *found))
+				{
+					if(found->origptr && *found->origptr == NULL)
+						*found->origptr = (void *)GetProcAddress(mod, func);
+
+					return (FARPROC)found->hookptr;
+				}
+			}
+		}
+	}
+
+	return GetProcAddress(mod, func);
+}
+
+void Win32_IAT_BeginHooks()
+{
+	s_HookData = new CachedHookData;
+	RDCASSERT(s_HookData->DllHooks.empty());
+	s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back(FunctionHook("LoadLibraryA", NULL, &Hooked_LoadLibraryA));
+	s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back(FunctionHook("LoadLibraryW", NULL, &Hooked_LoadLibraryW));
+	s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back(FunctionHook("LoadLibraryExA", NULL, &Hooked_LoadLibraryExA));
+	s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back(FunctionHook("LoadLibraryExW", NULL, &Hooked_LoadLibraryExW));
+	s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back(FunctionHook("GetProcAddress", NULL, &Hooked_GetProcAddress));
+	
+	GetModuleHandleEx(
+		GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS|GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+		(LPCTSTR)&s_HookData,
+		&s_HookData->module);
+
+	for(auto it=s_HookData->DllHooks.begin(); it != s_HookData->DllHooks.end(); ++it)
+		for(size_t i=0; i < it->second.FunctionHooks.size(); i++)
+			it->second.FunctionHooks[i].excludeModule = s_HookData->module;
+}
+
+// hook all functions for currently loaded modules.
+// some of these hooks (as above) will hook LoadLibrary/GetProcAddress, to protect
+void Win32_IAT_EndHooks()
+{
+	for(auto it=s_HookData->DllHooks.begin(); it != s_HookData->DllHooks.end(); ++it)
+		std::sort(it->second.FunctionHooks.begin(), it->second.FunctionHooks.end());
+
+	HANDLE hModuleSnap = INVALID_HANDLE_VALUE;
+
+	// up to 10 retries
+	for(int i=0; i < 10; i++)
+	{
+		hModuleSnap = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE, GetCurrentProcessId()); 
+
+		if(hModuleSnap == INVALID_HANDLE_VALUE) 
+		{
+			DWORD err = GetLastError();
+
+			RDCWARN("CreateToolhelp32Snapshot() -> 0x%08x", err);
+
+			// retry if error is ERROR_BAD_LENGTH
+			if(err == ERROR_BAD_LENGTH)
+				continue;
+		} 
+
+		// didn't retry, or succeeded
+		break;
+	}
+	
+	if(hModuleSnap == INVALID_HANDLE_VALUE) 
+	{
+		RDCERR("Couldn't create toolhelp dump of modules in process");
+		return;
+	} 
+
+#ifdef UNICODE
+#undef MODULEENTRY32
+#undef Module32First
+#undef Module32Next
+#endif
+ 
+  MODULEENTRY32 me32; 
+	RDCEraseEl(me32);
+  me32.dwSize = sizeof(MODULEENTRY32); 
+ 
+	BOOL success = Module32First(hModuleSnap, &me32);
+
+  if(success == FALSE) 
+  { 
+		DWORD err = GetLastError();
+
+		RDCERR("Couldn't get first module in process: 0x%08x", err);
+    CloseHandle(hModuleSnap);
+		return;
+  }
+
+	uintptr_t ret = 0;
+
+	int numModules = 0;
+ 
+  do
+  {
+		s_HookData->ApplyHooks(me32.szModule, me32.hModule);
+  } while(ret == 0 && Module32Next(hModuleSnap, &me32)); 
+
+  CloseHandle(hModuleSnap); 
+}
+
+bool Win32_IAT_Hook(void **orig_function_ptr, const char *module_name, const char *function, void *destination_function_ptr)
+{
+	if(!_stricmp(module_name, "kernel32.dll"))
+	{
+		if(!strcmp(function, "LoadLibraryA") ||
+			!strcmp(function, "LoadLibraryW") ||
+			!strcmp(function, "LoadLibraryExA") ||
+			!strcmp(function, "LoadLibraryExW") ||
+			!strcmp(function, "GetProcAddress"))
+		{
+			RDCERR("Cannot hook LoadLibrary* or GetProcAddress, as these are hooked internally");
+			return false;
+		}
+	}
+	s_HookData->DllHooks[strlower(string(module_name))].FunctionHooks.push_back(FunctionHook(function, orig_function_ptr, destination_function_ptr));
+	return true;
+}

renderdoc/os/win32/win32_hook.h

@@ -0,0 +1,29 @@
+/******************************************************************************
+ * The MIT License (MIT)
+ * 
+ * Copyright (c) 2014 Crytek
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ ******************************************************************************/
+
+#pragma once
+
+bool Win32_IAT_Hook(void **orig_function_ptr, const char *module_name, const char *function, void *destination_function_ptr);
+void Win32_IAT_BeginHooks();
+void Win32_IAT_EndHooks();

renderdoc/os/win32/win32_process.cpp

@@ -292,6 +292,7 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, const wchar_t *logfile, const
 	}
 #endif
 
+#if USE_MHOOK
 	// misc
 	InjectDLL(hProcess, L"kernel32.dll");
 
@@ -303,6 +304,7 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, const wchar_t *logfile, const
 	// OpenGL
 	InjectDLL(hProcess, L"opengl32.dll");
 	InjectDLL(hProcess, L"gdi32.dll");
+#endif
 
 	InjectDLL(hProcess, renderdocPath);
 

renderdoc/os/win32_specific.h

@@ -47,3 +47,8 @@ namespace Threading
 {
 	typedef CriticalSectionTemplate<CRITICAL_SECTION> CriticalSection;
 };
+
+// Hooking configuration
+#define USE_MHOOK 0
+#define USE_IAT_HOOK 1
+

renderdoc/renderdoc.vcxproj

@@ -275,6 +275,7 @@
     <ClInclude Include="maths\quat.h" />
     <ClInclude Include="maths\vec.h" />
     <ClInclude Include="os\os_specific.h" />
+    <ClInclude Include="os\win32\win32_hook.h" />
     <ClInclude Include="os\win32_specific.h" />
     <ClInclude Include="replay\basic_types.h" />
     <ClInclude Include="replay\capture_options.h" />
@@ -356,6 +357,7 @@
     <ClCompile Include="maths\matrix.cpp" />
     <ClCompile Include="os\os_specific.cpp" />
     <ClCompile Include="os\win32\win32_callstack.cpp" />
+    <ClCompile Include="os\win32\win32_hook.cpp" />
     <ClCompile Include="os\win32\win32_network.cpp" />
     <ClCompile Include="os\win32\win32_process.cpp" />
     <ClCompile Include="os\win32\win32_shellext.cpp" />

renderdoc/renderdoc.vcxproj.filters

@@ -288,6 +288,9 @@
     <ClInclude Include="driver\gl\gl_renderstate.h">
       <Filter>Drivers\OpenGL</Filter>
     </ClInclude>
+    <ClInclude Include="os\win32\win32_hook.h">
+      <Filter>OS\Win32</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="maths\camera.cpp">
@@ -506,6 +509,9 @@
     <ClCompile Include="driver\gl\wrappers\gl_query_funcs.cpp">
       <Filter>Drivers\OpenGL\Function Wrappers</Filter>
     </ClCompile>
+    <ClCompile Include="os\win32\win32_hook.cpp">
+      <Filter>OS\Win32</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="os\win32\comexport.def">