Hook initialisation can't fail, because it's just a registration

* Hooking no longer does anything active immediately when initialised, it just
  registers a function to be hooked, which is then deferred and/or continually
  re-hooked depending on platform-specific functionality.
This commit is contained in:
baldurk
2018-06-26 12:54:31 +01:00
parent 6a30e63a46
commit edd6ab4faa
11 changed files with 73 additions and 192 deletions
+6 -22
View File
@@ -28,8 +28,6 @@
#include "driver/ihv/amd/official/DXExt/AmdDxExtApi.h"
#include "hooks/hooks.h"
#define DLL_NAME "d3d11.dll"
ID3D11Resource *UnwrapDXResource(void *dxObject);
ID3DDevice *GetD3D11DeviceIfAlloc(IUnknown *dev)
@@ -111,16 +109,9 @@ typedef NVENCSTATUS(NVENCAPI *PFN_NvEncodeAPICreateInstance)(NV_ENCODE_API_FUNCT
class D3D11Hook : LibraryHook
{
public:
D3D11Hook()
{
m_HasHooks = false;
m_InsideCreate = false;
}
D3D11Hook() { m_InsideCreate = false; }
bool CreateHooks(const char *libName)
{
bool success = true;
WrappedIDXGISwapChain4::RegisterD3DDeviceCallback(GetD3D11DeviceIfAlloc);
// also require d3dcompiler_??.dll
@@ -130,12 +121,12 @@ public:
return false;
}
success &= CreateDevice.Initialize("D3D11CreateDevice", DLL_NAME, D3D11CreateDevice_hook);
success &= CreateDeviceAndSwapChain.Initialize("D3D11CreateDeviceAndSwapChain", DLL_NAME,
D3D11CreateDeviceAndSwapChain_hook);
CreateDevice.Initialize("D3D11CreateDevice", "d3d11.dll", D3D11CreateDevice_hook);
CreateDeviceAndSwapChain.Initialize("D3D11CreateDeviceAndSwapChain", "d3d11.dll",
D3D11CreateDeviceAndSwapChain_hook);
// these are not required for success, but opportunistic to prevent AMD extensions from
// activating and causing later crashes when not replayed correctly
// these are hooked to prevent AMD extensions from activating and causing later crashes when not
// replayed correctly
#if ENABLED(RDOC_X64)
AmdCreate11.Initialize("AmdDxExtCreate11", "atidxx64.dll", AmdCreate11_hook);
#else
@@ -160,19 +151,12 @@ public:
NvEncodeAPICreateInstance_hook);
#endif
if(!success)
return false;
m_HasHooks = true;
return true;
}
private:
static D3D11Hook d3d11hooks;
bool m_HasHooks;
Hook<PFN_D3D11_CREATE_DEVICE_AND_SWAP_CHAIN> CreateDeviceAndSwapChain;
Hook<PFN_D3D11_CREATE_DEVICE> CreateDevice;
+9 -33
View File
@@ -28,8 +28,6 @@
#include "d3d12_command_queue.h"
#include "d3d12_device.h"
#define DLL_NAME "d3d12.dll"
typedef HRESULT(WINAPI *PFN_D3D12_ENABLE_EXPERIMENTAL_FEATURES)(UINT NumFeatures, const IID *pIIDs,
void *pConfigurationStructs,
UINT *pConfigurationStructSizes);
@@ -88,16 +86,8 @@ public:
class D3D12Hook : LibraryHook
{
public:
D3D12Hook()
{
m_HasHooks = false;
m_InsideCreate = false;
}
bool CreateHooks(const char *libName)
{
bool success = true;
WrappedIDXGISwapChain4::RegisterD3DDeviceCallback(GetD3D12DeviceIfAlloc);
// also require d3dcompiler_??.dll
@@ -107,16 +97,10 @@ public:
return false;
}
success &= CreateDevice.Initialize("D3D12CreateDevice", DLL_NAME, D3D12CreateDevice_hook);
success &= GetDebugInterface.Initialize("D3D12GetDebugInterface", DLL_NAME,
D3D12GetDebugInterface_hook);
success &= EnableExperimentalFeatures.Initialize("D3D12EnableExperimentalFeatures", DLL_NAME,
D3D12EnableExperimentalFeatures_hook);
if(!success)
return false;
m_HasHooks = true;
CreateDevice.Initialize("D3D12CreateDevice", "d3d12.dll", D3D12CreateDevice_hook);
GetDebugInterface.Initialize("D3D12GetDebugInterface", "d3d12.dll", D3D12GetDebugInterface_hook);
EnableExperimentalFeatures.Initialize("D3D12EnableExperimentalFeatures", "d3d12.dll",
D3D12EnableExperimentalFeatures_hook);
return true;
}
@@ -124,14 +108,12 @@ public:
private:
static D3D12Hook d3d12hooks;
bool m_HasHooks;
Hook<PFN_D3D12_GET_DEBUG_INTERFACE> GetDebugInterface;
Hook<PFN_D3D12_CREATE_DEVICE> CreateDevice;
Hook<PFN_D3D12_ENABLE_EXPERIMENTAL_FEATURES> EnableExperimentalFeatures;
// re-entrancy detection (can happen in rare cases with e.g. fraps)
bool m_InsideCreate;
bool m_InsideCreate = false;
HRESULT Create_Internal(IUnknown *pAdapter, D3D_FEATURE_LEVEL MinimumFeatureLevel, REFIID riid,
void **ppDevice)
@@ -140,22 +122,16 @@ private:
// special. Just grab the trampolined function and call it.
if(m_InsideCreate)
{
PFN_D3D12_CREATE_DEVICE createFunc = NULL;
PFN_D3D12_CREATE_DEVICE createFunc = CreateDevice();
// shouldn't ever get in here if we're in the case without hooks but let's be safe.
if(m_HasHooks)
{
createFunc = CreateDevice();
}
else
if(!createFunc)
{
HMODULE d3d12 = GetModuleHandleA("d3d12.dll");
if(d3d12)
{
createFunc = (PFN_D3D12_CREATE_DEVICE)GetProcAddress(d3d12, "D3D12CreateDevice");
}
else
if(!createFunc)
{
RDCERR("Something went seriously wrong, d3d12.dll couldn't be loaded!");
return E_UNEXPECTED;
+1 -13
View File
@@ -26,24 +26,14 @@
#include "hooks/hooks.h"
#include "d3d8_device.h"
#define DLL_NAME "d3d8.dll"
typedef IDirect3D8 *(WINAPI *PFN_D3D8_CREATE)(UINT);
class D3D8Hook : LibraryHook
{
public:
D3D8Hook() { m_HasHooks = false; }
bool CreateHooks(const char *libName)
{
bool success = true;
success &= Create8.Initialize("Direct3DCreate8", DLL_NAME, Create8_hook);
if(!success)
return false;
m_HasHooks = true;
Create8.Initialize("Direct3DCreate8", "d3d8.dll", Create8_hook);
return true;
}
@@ -51,8 +41,6 @@ public:
private:
static D3D8Hook d3d8hooks;
bool m_HasHooks;
Hook<PFN_D3D8_CREATE> Create8;
static IDirect3D8 *WINAPI Create8_hook(UINT SDKVersion)
+6 -18
View File
@@ -29,8 +29,6 @@
#include "driver/dx/official/d3d9.h"
#include "d3d9_device.h"
#define DLL_NAME "d3d9.dll"
typedef int(WINAPI *PFN_BEGIN_EVENT)(DWORD, WCHAR *);
typedef int(WINAPI *PFN_END_EVENT)();
typedef int(WINAPI *PFN_SET_MARKER_EVENT)(DWORD, WCHAR *);
@@ -42,23 +40,15 @@ typedef IDirect3D9 *(WINAPI *PFN_D3D9_CREATE)(UINT);
class D3D9Hook : LibraryHook
{
public:
D3D9Hook() { m_HasHooks = false; }
bool CreateHooks(const char *libName)
{
bool success = true;
PERF_BeginEvent.Initialize("D3DPERF_BeginEvent", "d3d9.dll", PERF_BeginEvent_hook);
PERF_EndEvent.Initialize("D3DPERF_EndEvent", "d3d9.dll", PERF_EndEvent_hook);
PERF_SetMarker.Initialize("D3DPERF_SetMarker", "d3d9.dll", PERF_SetMarker_hook);
PERF_SetOptions.Initialize("D3DPERF_SetOptions", "d3d9.dll", PERF_SetOptions_hook);
PERF_GetStatus.Initialize("D3DPERF_GetStatus", "d3d9.dll", PERF_GetStatus_hook);
success &= PERF_BeginEvent.Initialize("D3DPERF_BeginEvent", DLL_NAME, PERF_BeginEvent_hook);
success &= PERF_EndEvent.Initialize("D3DPERF_EndEvent", DLL_NAME, PERF_EndEvent_hook);
success &= PERF_SetMarker.Initialize("D3DPERF_SetMarker", DLL_NAME, PERF_SetMarker_hook);
success &= PERF_SetOptions.Initialize("D3DPERF_SetOptions", DLL_NAME, PERF_SetOptions_hook);
success &= PERF_GetStatus.Initialize("D3DPERF_GetStatus", DLL_NAME, PERF_GetStatus_hook);
success &= Create9.Initialize("Direct3DCreate9", DLL_NAME, Create9_hook);
if(!success)
return false;
m_HasHooks = true;
Create9.Initialize("Direct3DCreate9", "d3d9.dll", Create9_hook);
return true;
}
@@ -66,8 +56,6 @@ public:
private:
static D3D9Hook d3d9hooks;
bool m_HasHooks;
// D3DPERF api
Hook<PFN_BEGIN_EVENT> PERF_BeginEvent;
Hook<PFN_END_EVENT> PERF_EndEvent;
+5 -13
View File
@@ -27,8 +27,6 @@
#include "hooks/hooks.h"
#include "dxgi_wrapped.h"
#define DLL_NAME "dxgi.dll"
typedef HRESULT(WINAPI *PFN_CREATE_DXGI_FACTORY)(REFIID, void **);
typedef HRESULT(WINAPI *PFN_CREATE_DXGI_FACTORY2)(UINT, REFIID, void **);
typedef HRESULT(WINAPI *PFN_GET_DEBUG_INTERFACE)(REFIID, void **);
@@ -76,17 +74,11 @@ class DXGIHook : LibraryHook
public:
bool CreateHooks(const char *libName)
{
bool success = true;
success &= CreateDXGIFactory.Initialize("CreateDXGIFactory", DLL_NAME, CreateDXGIFactory_hook);
success &= CreateDXGIFactory1.Initialize("CreateDXGIFactory1", DLL_NAME, CreateDXGIFactory1_hook);
// don't mind if these dosn't succeed
CreateDXGIFactory2.Initialize("CreateDXGIFactory2", DLL_NAME, CreateDXGIFactory2_hook);
GetDebugInterface.Initialize("DXGIGetDebugInterface", DLL_NAME, DXGIGetDebugInterface_hook);
GetDebugInterface1.Initialize("DXGIGetDebugInterface1", DLL_NAME, DXGIGetDebugInterface1_hook);
if(!success)
return false;
CreateDXGIFactory.Initialize("CreateDXGIFactory", "dxgi.dll", CreateDXGIFactory_hook);
CreateDXGIFactory1.Initialize("CreateDXGIFactory1", "dxgi.dll", CreateDXGIFactory1_hook);
CreateDXGIFactory2.Initialize("CreateDXGIFactory2", "dxgi.dll", CreateDXGIFactory2_hook);
GetDebugInterface.Initialize("DXGIGetDebugInterface", "dxgi.dll", DXGIGetDebugInterface_hook);
GetDebugInterface1.Initialize("DXGIGetDebugInterface1", "dxgi.dll", DXGIGetDebugInterface1_hook);
return true;
}
+2 -14
View File
@@ -49,8 +49,6 @@ class OpenGLHook : LibraryHook, public GLPlatform
public:
OpenGLHook()
{
m_HasHooks = false;
m_GLDriver = NULL;
m_PopulatedHooks = false;
@@ -67,12 +65,7 @@ public:
if(libName)
PosixHookLibrary("libGL.so", &libHooked);
bool success = SetupHooks();
if(!success)
return false;
m_HasHooks = true;
SetupHooks();
return true;
}
@@ -386,12 +379,9 @@ public:
map<XID, XID> m_GLXWindowMap;
bool m_PopulatedHooks;
bool m_HasHooks;
bool SetupHooks()
void SetupHooks()
{
bool success = true;
if(glXGetProcAddress_real == NULL)
glXGetProcAddress_real =
(PFNGLXGETPROCADDRESSPROC)dlsym(libGLdlsymHandle, "glXGetProcAddress");
@@ -436,8 +426,6 @@ public:
if(glXCreateContextAttribsARB_real == NULL && glXGetProcAddress_real)
glXCreateContextAttribsARB_real = (PFNGLXCREATECONTEXTATTRIBSARBPROC)glXGetProcAddress_real(
(const GLubyte *)"glXCreateContextAttribsARB");
return success;
}
bool PopulateHooks();
+27 -46
View File
@@ -31,14 +31,9 @@
#include "hooks/hooks.h"
#include "strings/string_utils.h"
#define DLL_NAME "opengl32.dll"
#define HookInit(function) \
bool CONCAT(function, _success) = \
CONCAT(function, _hook).Initialize(STRINGIZE(function), DLL_NAME, CONCAT(function, _hooked)); \
if(!CONCAT(function, _success)) \
RDCWARN("Couldn't hook %s", STRINGIZE(function)); \
success &= CONCAT(function, _success); \
#define HookInit(function) \
CONCAT(function, _hook) \
.Initialize(STRINGIZE(function), "opengl32.dll", CONCAT(function, _hooked)); \
GL.function = CONCAT(function, _hook)();
#define HookExtension(funcPtrType, function) \
@@ -669,8 +664,6 @@ public:
{
m_GLDriver = NULL;
m_HasHooks = false;
m_HaveContextCreation = false;
m_PopulatedHooks = false;
@@ -682,12 +675,7 @@ public:
~OpenGLHook() { delete m_GLDriver; }
bool CreateHooks(const char *libName)
{
bool success = SetupHooks();
if(!success)
return false;
m_HasHooks = true;
SetupHooks();
return true;
}
@@ -960,7 +948,7 @@ private:
if(glhooks.wglGetProcAddress_hook() == NULL)
glhooks.wglGetProcAddress_hook.SetFuncPtr(
Process::GetFunctionAddress(DLL_NAME, "wglGetProcAddress"));
Process::GetFunctionAddress(Process::LoadModule("opengl32.dll"), "wglGetProcAddress"));
if(glhooks.wglGetPixelFormatAttribivARB_realfunc == NULL)
glhooks.wglGetProcAddress_hook()("wglGetPixelFormatAttribivARB");
@@ -1361,46 +1349,39 @@ private:
WrappedOpenGL *m_GLDriver;
bool m_PopulatedHooks;
bool m_HasHooks;
set<HGLRC> m_Contexts;
bool SetupHooks()
void SetupHooks()
{
bool success = true;
wglCreateContext_hook.Initialize("wglCreateContext", "opengl32.dll", wglCreateContext_hooked);
wglDeleteContext_hook.Initialize("wglDeleteContext", "opengl32.dll", wglDeleteContext_hooked);
wglCreateLayerContext_hook.Initialize("wglCreateLayerContext", "opengl32.dll",
wglCreateLayerContext_hooked);
wglMakeCurrent_hook.Initialize("wglMakeCurrent", "opengl32.dll", wglMakeCurrent_hooked);
success &=
wglCreateContext_hook.Initialize("wglCreateContext", DLL_NAME, wglCreateContext_hooked);
success &=
wglDeleteContext_hook.Initialize("wglDeleteContext", DLL_NAME, wglDeleteContext_hooked);
success &= wglCreateLayerContext_hook.Initialize("wglCreateLayerContext", DLL_NAME,
wglCreateLayerContext_hooked);
success &= wglMakeCurrent_hook.Initialize("wglMakeCurrent", DLL_NAME, wglMakeCurrent_hooked);
success &=
wglGetProcAddress_hook.Initialize("wglGetProcAddress", DLL_NAME, wglGetProcAddress_hooked);
success &= wglSwapBuffers_hook.Initialize("wglSwapBuffers", DLL_NAME, wglSwapBuffers_hooked);
success &= wglSwapLayerBuffers_hook.Initialize("wglSwapLayerBuffers", DLL_NAME,
wglSwapLayerBuffers_hooked);
success &= wglSwapMultipleBuffers_hook.Initialize("wglSwapMultipleBuffers", DLL_NAME,
wglSwapMultipleBuffers_hooked);
success &= SwapBuffers_hook.Initialize("SwapBuffers", "gdi32.dll", SwapBuffers_hooked);
success &= ChangeDisplaySettingsA_hook.Initialize("ChangeDisplaySettingsA", "user32.dll",
ChangeDisplaySettingsA_hooked);
success &= ChangeDisplaySettingsW_hook.Initialize("ChangeDisplaySettingsW", "user32.dll",
ChangeDisplaySettingsW_hooked);
success &= ChangeDisplaySettingsExA_hook.Initialize("ChangeDisplaySettingsExA", "user32.dll",
ChangeDisplaySettingsExA_hooked);
success &= ChangeDisplaySettingsExW_hook.Initialize("ChangeDisplaySettingsExW", "user32.dll",
ChangeDisplaySettingsExW_hooked);
wglGetProcAddress_hook.Initialize("wglGetProcAddress", "opengl32.dll", wglGetProcAddress_hooked);
wglSwapBuffers_hook.Initialize("wglSwapBuffers", "opengl32.dll", wglSwapBuffers_hooked);
wglSwapLayerBuffers_hook.Initialize("wglSwapLayerBuffers", "opengl32.dll",
wglSwapLayerBuffers_hooked);
wglSwapMultipleBuffers_hook.Initialize("wglSwapMultipleBuffers", "opengl32.dll",
wglSwapMultipleBuffers_hooked);
SwapBuffers_hook.Initialize("SwapBuffers", "gdi32.dll", SwapBuffers_hooked);
ChangeDisplaySettingsA_hook.Initialize("ChangeDisplaySettingsA", "user32.dll",
ChangeDisplaySettingsA_hooked);
ChangeDisplaySettingsW_hook.Initialize("ChangeDisplaySettingsW", "user32.dll",
ChangeDisplaySettingsW_hooked);
ChangeDisplaySettingsExA_hook.Initialize("ChangeDisplaySettingsExA", "user32.dll",
ChangeDisplaySettingsExA_hooked);
ChangeDisplaySettingsExW_hook.Initialize("ChangeDisplaySettingsExW", "user32.dll",
ChangeDisplaySettingsExW_hooked);
DLLExportHooks();
return success;
}
bool PopulateHooks()
{
void *moduleHandle = Process::LoadModule(DLL_NAME);
void *moduleHandle = Process::LoadModule("opengl32.dll");
if(wglGetProcAddress_hook() == NULL)
wglGetProcAddress_hook.SetFuncPtr(
+2 -2
View File
@@ -44,11 +44,11 @@ public:
~Hook() {}
FuncType operator()() { return (FuncType)orig_funcptr; }
void SetFuncPtr(void *ptr) { orig_funcptr = ptr; }
bool Initialize(const char *function, const char *module_name, void *destination_function_ptr)
void Initialize(const char *function, const char *module_name, void *destination_function_ptr)
{
orig_funcptr = Process::GetFunctionAddress(Process::LoadModule(module_name), function);
return Win32_IAT_Hook(&orig_funcptr, module_name, function, destination_function_ptr);
Win32_IAT_Hook(&orig_funcptr, module_name, function, destination_function_ptr);
}
private:
+12 -27
View File
@@ -29,8 +29,6 @@
#include "hooks/hooks.h"
#include "strings/string_utils.h"
#define DLL_NAME "kernel32.dll"
typedef int(WSAAPI *PFN_WSASTARTUP)(__in WORD wVersionRequested, __out LPWSADATA lpWSAData);
typedef int(WSAAPI *PFN_WSACLEANUP)();
@@ -75,30 +73,27 @@ class SysHook : LibraryHook
public:
SysHook()
{
m_HasHooks = false;
// we start with a refcount of 1 because we initialise WSA ourselves for our own sockets.
m_WSARefCount = 1;
}
bool CreateHooks(const char *libName)
{
bool success = true;
// we want to hook CreateProcess purely so that we can recursively insert our hooks (if we so
// wish)
success &= CreateProcessA.Initialize("CreateProcessA", "kernel32.dll", CreateProcessA_hook);
success &= CreateProcessW.Initialize("CreateProcessW", "kernel32.dll", CreateProcessW_hook);
CreateProcessA.Initialize("CreateProcessA", "kernel32.dll", CreateProcessA_hook);
CreateProcessW.Initialize("CreateProcessW", "kernel32.dll", CreateProcessW_hook);
success &= CreateProcessAsUserA.Initialize("CreateProcessAsUserA", "advapi32.dll",
CreateProcessAsUserA_hook);
success &= CreateProcessAsUserW.Initialize("CreateProcessAsUserW", "advapi32.dll",
CreateProcessAsUserW_hook);
CreateProcessAsUserA.Initialize("CreateProcessAsUserA", "advapi32.dll",
CreateProcessAsUserA_hook);
CreateProcessAsUserW.Initialize("CreateProcessAsUserW", "advapi32.dll",
CreateProcessAsUserW_hook);
success &= CreateProcessWithLogonW.Initialize("CreateProcessWithLogonW", "advapi32.dll",
CreateProcessWithLogonW_hook);
CreateProcessWithLogonW.Initialize("CreateProcessWithLogonW", "advapi32.dll",
CreateProcessWithLogonW_hook);
// handle API set exports if they exist. These don't really exist so we don't have to worry
// about
// double hooking, and also they call into the 'real' implementation in kernelbase.dll
// about double hooking, and also they call into the 'real' implementation in kernelbase.dll
API110CreateProcessA.Initialize("CreateProcessA", "api-ms-win-core-processthreads-l1-1-0.dll",
API110CreateProcessA_hook);
API110CreateProcessW.Initialize("CreateProcessW", "api-ms-win-core-processthreads-l1-1-0.dll",
@@ -123,28 +118,18 @@ public:
"api-ms-win-core-processthreads-l1-1-0.dll",
API112CreateProcessAsUserW_hook);
success &= WSAStartup.Initialize("WSAStartup", "ws2_32.dll", WSAStartup_hook);
success &= WSACleanup.Initialize("WSACleanup", "ws2_32.dll", WSACleanup_hook);
// we start with a refcount of 1 because we initialise WSA ourselves for our own sockets.
m_WSARefCount = 1;
WSAStartup.Initialize("WSAStartup", "ws2_32.dll", WSAStartup_hook);
WSACleanup.Initialize("WSACleanup", "ws2_32.dll", WSACleanup_hook);
m_RecurseSlot = Threading::AllocateTLSSlot();
Threading::SetTLSValue(m_RecurseSlot, NULL);
if(!success)
return false;
m_HasHooks = true;
return true;
}
private:
static SysHook syshooks;
bool m_HasHooks;
int m_WSARefCount;
uint64_t m_RecurseSlot = 0;
+2 -3
View File
@@ -845,7 +845,7 @@ void Win32_IAT_RemoveHooks()
}
}
bool Win32_IAT_Hook(void **orig_function_ptr, const char *module_name, const char *function,
void Win32_IAT_Hook(void **orig_function_ptr, const char *module_name, const char *function,
void *destination_function_ptr)
{
if(!_stricmp(module_name, "kernel32.dll"))
@@ -855,12 +855,11 @@ bool Win32_IAT_Hook(void **orig_function_ptr, const char *module_name, const cha
!strcmp(function, "GetProcAddress"))
{
RDCERR("Cannot hook LoadLibrary* or GetProcAddress, as these are hooked internally");
return false;
return;
}
}
s_HookData->DllHooks[strlower(string(module_name))].FunctionHooks.push_back(
FunctionHook(function, orig_function_ptr, destination_function_ptr));
return true;
}
bool Win32_HookDetect(const char *identifier)
+1 -1
View File
@@ -25,7 +25,7 @@
#pragma once
bool Win32_IAT_Hook(void **orig_function_ptr, const char *module_name, const char *function,
void Win32_IAT_Hook(void **orig_function_ptr, const char *module_name, const char *function,
void *destination_function_ptr);
void Win32_IAT_BeginHooks();
void Win32_IAT_EndHooks();