Use minimal function hooking to allow dynamic D3D12 runtime selection

* By intercepting D3D12.dll's load of D3D12Core.dll and wrapping the COM
  interface it gets back, we can load the D3D12Core.dll that we desired.
* As a default, we load the D3D12Core.dll embedded in the capture. There's also
  a config setting to allow the user to override this, though in principle this
  shouldn't be needed.
* We only do all this if the SDK version used to capture is higher than that
  normally available on the system. So if a new runtime isn't in use, we won't
  do any hooking at all.
This commit is contained in:
baldurk committed 2021-04-22 15:52:15 +01:00
1 parent 2b74938ccd
commit f4d4c1862e
6 files changed
+394 -45

No files matched your search

+3
View File
@@ -59,6 +59,9 @@ struct D3D12MarkerRegion
bool EnableD3D12DebugLayer(PFN_D3D12_GET_DEBUG_INTERFACE getDebugInterface = NULL);
HRESULT EnumAdapterByLuid(IDXGIFactory1 *factory, LUID luid, IDXGIAdapter **pAdapter);
void D3D12_PrepareReplaySDKVersion(UINT SDKVersion, bytebuf d3d12core, HMODULE d3d12lib);
void D3D12_CleanupReplaySDK();
inline void SetObjName(ID3D12Object *obj, const rdcstr &utf8name)
{
obj->SetName(StringFormat::UTF82Wide(utf8name).c_str());
+24 -3
View File
@@ -74,6 +74,7 @@ void D3D12Replay::Shutdown()
// the this pointer is free'd after this point
FreeLibrary(D3D12Lib);
D3D12_CleanupReplaySDK();
}
void D3D12Replay::Initialise(IDXGIFactory1 *factory)
@@ -3921,6 +3922,7 @@ ReplayStatus D3D12_CreateReplayDevice(RDCFile *rdc, const ReplayOptions &opts, I
}
D3D12InitParams initParams;
bytebuf D3D12Core;
uint64_t ver = D3D12InitParams::CurrentVersion;
@@ -3968,6 +3970,28 @@ ReplayStatus D3D12_CreateReplayDevice(RDCFile *rdc, const ReplayOptions &opts, I
ToStr(GPUVendorFromPCIVendor(initParams.AdapterDesc.VendorId)).c_str(),
initParams.AdapterDesc.Description);
sectionIdx = rdc->SectionIndex(SectionType::D3D12Core);
if(sectionIdx >= 0)
{
SectionProperties props = rdc->GetSectionProperties(sectionIdx);
reader = rdc->ReadSection(sectionIdx);
D3D12Core.resize(props.uncompressedSize);
reader->Read(D3D12Core.data(), props.uncompressedSize);
RDCASSERT(reader->AtEnd());
delete reader;
}
}
if(initParams.MinimumFeatureLevel < D3D_FEATURE_LEVEL_11_0)
initParams.MinimumFeatureLevel = D3D_FEATURE_LEVEL_11_0;
D3D12_PrepareReplaySDKVersion(initParams.SDKVersion, D3D12Core, D3D12Lib);
if(rdc)
{
using PFN_ENABLE_EXPERIMENTAL = decltype(&D3D12EnableExperimentalFeatures);
PFN_ENABLE_EXPERIMENTAL EnableExperimental =
@@ -3987,9 +4011,6 @@ ReplayStatus D3D12_CreateReplayDevice(RDCFile *rdc, const ReplayOptions &opts, I
}
}
if(initParams.MinimumFeatureLevel < D3D_FEATURE_LEVEL_11_0)
initParams.MinimumFeatureLevel = D3D_FEATURE_LEVEL_11_0;
const bool isProxy = (rdc == NULL);
AMDRGPControl *rgp = NULL;
+271
View File
@@ -0,0 +1,271 @@
/******************************************************************************
* The MIT License (MIT)
*
* Copyright (c) 2021 Baldur Karlsson
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
******************************************************************************/
#include "core/settings.h"
#include "hooks/hooks.h"
#include "strings/string_utils.h"
#include "d3d12_common.h"
RDOC_CONFIG(rdcstr, D3D12_D3D12CorePath, "",
"The location of the D3D12Core library. This path should be the directory that "
"contains the D3D12Core.dll that you want to use.");
// special hooking functions exposed on windows for hooking while in replay mode
void Win32_RegisterManualModuleHooking();
void Win32_InterceptLibraryLoads(std::function<HMODULE(const rdcstr &, HANDLE, DWORD)> callback);
void Win32_ManualHookModule(rdcstr modName, HMODULE module);
namespace
{
DWORD SystemCoreVersion;
rdcstr D3D12Core_Override_Path;
rdcstr D3D12Core_Temp_Path;
};
MIDL_INTERFACE("DFAFDD2C-355F-4CB3-A8B2-EA7F9260148B")
ID3D12CoreModule : public IUnknown
{
public:
virtual DWORD STDMETHODCALLTYPE LOEnter(void) = 0;
virtual DWORD STDMETHODCALLTYPE LOLeave(void) = 0;
virtual DWORD STDMETHODCALLTYPE LOTryEnter(void) = 0;
virtual HRESULT STDMETHODCALLTYPE Initialize(DWORD, LPCSTR) = 0;
virtual DWORD STDMETHODCALLTYPE GetSDKVersion(void) = 0;
virtual HRESULT STDMETHODCALLTYPE GetDllExports(void *) = 0;
};
struct WrappedCoreModule : public ID3D12CoreModule
{
unsigned int m_iRefcount = 0;
ID3D12CoreModule *m_pReal = NULL;
WrappedCoreModule(ID3D12CoreModule *real) : m_iRefcount(1), m_pReal(real) {}
~WrappedCoreModule() { SAFE_RELEASE(m_pReal); }
//////////////////////////////
// implement IUnknown
HRESULT STDMETHODCALLTYPE QueryInterface(REFIID riid, void **ppvObject) { return E_NOINTERFACE; }
ULONG STDMETHODCALLTYPE AddRef()
{
InterlockedIncrement(&m_iRefcount);
return m_iRefcount;
}
ULONG STDMETHODCALLTYPE Release()
{
unsigned int ret = InterlockedDecrement(&m_iRefcount);
if(ret == 0)
delete this;
return ret;
}
//////////////////////////////
// implement ID3D12CoreModule
virtual DWORD STDMETHODCALLTYPE LOEnter(void) { return m_pReal->LOEnter(); }
virtual DWORD STDMETHODCALLTYPE LOLeave(void) { return m_pReal->LOLeave(); }
virtual DWORD STDMETHODCALLTYPE LOTryEnter(void) { return m_pReal->LOTryEnter(); }
virtual HRESULT STDMETHODCALLTYPE Initialize(DWORD version, LPCSTR unknown)
{
// D3D12 should always think it's loading the system core version
RDCASSERT(version == SystemCoreVersion);
// initialize the actual core (which might in some cases be the system version) with its own
// version so it doesn't complain
return m_pReal->Initialize(m_pReal->GetSDKVersion(), unknown);
}
virtual DWORD STDMETHODCALLTYPE GetSDKVersion(void) { return SystemCoreVersion; }
virtual HRESULT STDMETHODCALLTYPE GetDllExports(void *a) { return m_pReal->GetDllExports(a); }
};
using PFN_D3D12_GET_INTERFACE = decltype(&D3D12GetInterface);
HookedFunction<PFN_D3D12_GET_INTERFACE> D3D12GetInterface_hook;
HMODULE Hooked_D3D12LoadLibrary(const rdcstr &filename, HANDLE h, DWORD flags)
{
// if we detect the intercepted call to D3D12Core.dll and we have a redirect path, load that
// redirect path instead
if(strlower(filename).contains("d3d12core.dll") && !D3D12Core_Override_Path.empty())
return LoadLibraryExW(StringFormat::UTF82Wide(D3D12Core_Override_Path).c_str(), NULL, flags);
return NULL;
}
HRESULT WINAPI Hooked_D3D12GetInterface(_In_ REFCLSID rclsid, _In_ REFIID riid,
_COM_Outptr_opt_ void **ppvDebug)
{
HRESULT ret = D3D12GetInterface_hook()(rclsid, riid, ppvDebug);
// intercept the interface with our own wrapper to ensure version checking is silenced
if(SUCCEEDED(ret) && riid == __uuidof(ID3D12CoreModule))
*ppvDebug = (ID3D12CoreModule *)(new WrappedCoreModule((ID3D12CoreModule *)*ppvDebug));
return ret;
}
void D3D12_PrepareReplaySDKVersion(UINT SDKVersion, bytebuf d3d12core_file, HMODULE d3d12lib)
{
// D3D12Core shouldn't be loaded at this point, but it might be due to bugs. If it is, we can't do
// anything to change it anymore so we have to just handle what we have
HMODULE D3D12Core = GetModuleHandleA("D3D12Core.dll");
if(D3D12Core != NULL)
{
DWORD *ver_ptr = (DWORD *)GetProcAddress(D3D12Core, "D3D12SDKVersion");
// if the core that's loaded is sufficient, don't show any warnings
if(ver_ptr && SDKVersion <= *ver_ptr)
return;
RDCWARN(
"D3D12Core.dll was already loaded before replay started. This may be caused by a D3D12 "
"runtime bug if the validation layers are enabled, that means D3D12 is never unloaded.");
if(ver_ptr)
RDCWARN("The existing D3D12Core.dll is version %u but this capture requires version %u",
*ver_ptr, SDKVersion);
else
RDCWARN("The existing D3D12Core.dll had an unknown version, this capture requires version %u",
SDKVersion);
return;
}
static bool core_fetched = false, hooks_applied = false;
// if we don't have the system core version yet
if(!core_fetched)
{
core_fetched = true;
// default to 0
SystemCoreVersion = 0;
// get the system path to load it explicitly
wchar_t sys_path[MAX_PATH + 16] = {};
GetSystemDirectoryW(sys_path, MAX_PATH);
wcscat_s(sys_path, L"\\D3D12Core.dll");
HMODULE real_sys = LoadLibraryW(sys_path);
if(real_sys)
{
DWORD *ver_ptr = (DWORD *)GetProcAddress(real_sys, "D3D12SDKVersion");
if(ver_ptr)
SystemCoreVersion = *ver_ptr;
else
RDCERR("D3D12Core.dll loaded from %ls doesn't have D3D12SDKVersion export!", sys_path);
FreeLibrary(real_sys);
RDCLOG("System D3D12 runtime is version %u", SystemCoreVersion);
}
else
{
RDCLOG("No system D3D12 runtime found at %ls.", sys_path);
// if the captured SDK version was greater than 1 then most likely this capture will fail to
// replay, but we can still try to replay (there's no guarantee a user actually used features
// exclusive to the new runtime).
if(SDKVersion > 1)
RDCWARN(
"Capture was made with runtime version %u but this system does not support D3D12 "
"runtimes, possible incompatibility");
}
}
// if the system doesn't have a core DLL we can't intercept and point to our own runtime, so just
// abort here before doing anything potentially dangerous below.
if(SystemCoreVersion == 0)
return;
// similarly, if the system version is enough then the user didn't use a new runtime (or they used
// what was at the time a new runtime but is now available in the system...), so also abort.
// That means we'll only do the interception & patching when we think it's really needed.
if(SDKVersion <= SystemCoreVersion)
return;
// finally we're at a point where we will hook to force the library we want.
if(!hooks_applied)
{
Win32_RegisterManualModuleHooking();
D3D12GetInterface_hook.Register("d3d12core.dll", "D3D12GetInterface", Hooked_D3D12GetInterface);
Win32_InterceptLibraryLoads(Hooked_D3D12LoadLibrary);
}
// we do this always, even if the hooks are already applied, because this module has possibly been
// reloaded and needs to be re-hooked each time
Win32_ManualHookModule("d3d12.dll", d3d12lib);
// *always* use the user's path if it exists
D3D12Core_Override_Path = D3D12_D3D12CorePath();
if(D3D12Core_Override_Path.empty() || !FileIO::exists(D3D12Core_Override_Path.c_str()))
{
if(d3d12core_file.empty())
{
RDCERR(
"No D3D12Core.dll embedded in capture but we need a newer one (version %u) to properly "
"replay this capture",
SDKVersion);
return;
}
// find an appropriate spot to write this file. Other instances of RenderDoc might be running so
// we try a few different variants
for(uint32_t i = 0; i < 32; i++)
{
rdcstr filename = StringFormat::Fmt("%s/RenderDoc/D3D12Core/%u.ver%u/D3D12Core.dll",
FileIO::GetTempFolderFilename().c_str(), i, SDKVersion);
FileIO::CreateParentDirectory(filename);
FILE *f = FileIO::fopen(filename.c_str(), FileIO::WriteBinary);
// if we can write to this file, we have exclusive use of it so let's write it and use it
if(f)
{
FileIO::fwrite(d3d12core_file.data(), 1, d3d12core_file.size(), f);
FileIO::fclose(f);
D3D12Core_Override_Path = filename;
D3D12Core_Temp_Path = filename;
break;
}
}
if(D3D12Core_Override_Path.empty() || !FileIO::exists(D3D12Core_Override_Path.c_str()))
{
RDCERR("Couldn't write embedded D3D12Core.dll to disk! system dll will be used");
}
}
}
void D3D12_CleanupReplaySDK()
{
if(!D3D12Core_Temp_Path.empty() && FileIO::exists(D3D12Core_Temp_Path.c_str()))
{
FileIO::Delete(D3D12Core_Temp_Path.c_str());
}
}
@@ -136,6 +136,7 @@
<ClCompile Include="d3d12_rendertexture.cpp" />
<ClCompile Include="d3d12_replay.cpp" />
<ClCompile Include="d3d12_resources.cpp" />
<ClCompile Include="d3d12_sdk_select.cpp" />
<ClCompile Include="d3d12_serialise.cpp" />
<ClCompile Include="d3d12_shaderdebug.cpp" />
<ClCompile Include="d3d12_shader_cache.cpp" />
@@ -201,5 +201,8 @@
<ClCompile Include="d3d12_device_wrap9.cpp">
<Filter>Wrapped</Filter>
</ClCompile>
<ClCompile Include="d3d12_sdk_select.cpp">
<Filter>Util</Filter>
</ClCompile>
</ItemGroup>
</Project>
+92 -42
View File
@@ -148,21 +148,17 @@ struct DllHookset
struct CachedHookData
{
CachedHookData()
{
ownmodule = NULL;
missedOrdinals = false;
RDCEraseEl(lowername);
}
bool hookAll = true;
std::map<rdcstr, DllHookset> DllHooks;
HMODULE ownmodule;
HMODULE ownmodule = NULL;
Threading::CriticalSection lock;
char lowername[512];
char lowername[512] = {};
std::set<rdcstr> ignores;
bool missedOrdinals;
bool missedOrdinals = false;
std::function<HMODULE(const rdcstr &, HANDLE, DWORD)> libraryIntercept;
int32_t posthooking = 0;
@@ -564,6 +560,9 @@ static void ForAllModules(std::function<void(const MODULEENTRY32 &me32)> callbac
static void HookAllModules()
{
if(!s_HookData->hookAll)
return;
ForAllModules(
[](const MODULEENTRY32 &me32) { s_HookData->ApplyHooks(me32.szModule, me32.hModule); });
@@ -636,6 +635,15 @@ static bool IsAPISet(const char *filename)
HMODULE WINAPI Hooked_LoadLibraryExA(LPCSTR lpLibFileName, HANDLE fileHandle, DWORD flags)
{
bool dohook = true;
if(s_HookData->libraryIntercept)
{
HMODULE ret = s_HookData->libraryIntercept(lpLibFileName, fileHandle, flags);
if(ret)
return ret;
dohook = false;
}
if(flags == 0 && GetModuleHandleA(lpLibFileName))
dohook = false;
@@ -662,6 +670,16 @@ HMODULE WINAPI Hooked_LoadLibraryExA(LPCSTR lpLibFileName, HANDLE fileHandle, DW
HMODULE WINAPI Hooked_LoadLibraryExW(LPCWSTR lpLibFileName, HANDLE fileHandle, DWORD flags)
{
bool dohook = true;
if(s_HookData->libraryIntercept)
{
HMODULE ret =
s_HookData->libraryIntercept(StringFormat::Wide2UTF8(lpLibFileName), fileHandle, flags);
if(ret)
return ret;
dohook = false;
}
if(flags == 0 && GetModuleHandleW(lpLibFileName))
dohook = false;
@@ -814,6 +832,46 @@ FARPROC WINAPI Hooked_GetProcAddress(HMODULE mod, LPCSTR func)
return GetProcAddress(mod, func);
}
static void InitHookData()
{
if(!s_HookData)
{
s_HookData = new CachedHookData;
RDCASSERT(s_HookData->DllHooks.empty());
s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back(
FunctionHook("LoadLibraryA", NULL, &Hooked_LoadLibraryA));
s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back(
FunctionHook("LoadLibraryW", NULL, &Hooked_LoadLibraryW));
s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back(
FunctionHook("LoadLibraryExA", NULL, &Hooked_LoadLibraryExA));
s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back(
FunctionHook("LoadLibraryExW", NULL, &Hooked_LoadLibraryExW));
s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back(
FunctionHook("GetProcAddress", NULL, &Hooked_GetProcAddress));
for(const char *apiset :
{"api-ms-win-core-libraryloader-l1-1-0.dll", "api-ms-win-core-libraryloader-l1-1-1.dll",
"api-ms-win-core-libraryloader-l1-1-2.dll", "api-ms-win-core-libraryloader-l1-2-0.dll",
"api-ms-win-core-libraryloader-l1-2-1.dll"})
{
s_HookData->DllHooks[apiset].FunctionHooks.push_back(
FunctionHook("LoadLibraryA", NULL, &Hooked_LoadLibraryA));
s_HookData->DllHooks[apiset].FunctionHooks.push_back(
FunctionHook("LoadLibraryW", NULL, &Hooked_LoadLibraryW));
s_HookData->DllHooks[apiset].FunctionHooks.push_back(
FunctionHook("LoadLibraryExA", NULL, &Hooked_LoadLibraryExA));
s_HookData->DllHooks[apiset].FunctionHooks.push_back(
FunctionHook("LoadLibraryExW", NULL, &Hooked_LoadLibraryExW));
s_HookData->DllHooks[apiset].FunctionHooks.push_back(
FunctionHook("GetProcAddress", NULL, &Hooked_GetProcAddress));
}
GetModuleHandleEx(
GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
(LPCTSTR)&s_HookData, &s_HookData->ownmodule);
}
}
void LibraryHooks::RegisterFunctionHook(const char *libraryName, const FunctionHook &hook)
{
@@ -847,39 +905,7 @@ void LibraryHooks::IgnoreLibrary(const char *libraryName)
void LibraryHooks::BeginHookRegistration()
{
s_HookData = new CachedHookData;
RDCASSERT(s_HookData->DllHooks.empty());
s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back(
FunctionHook("LoadLibraryA", NULL, &Hooked_LoadLibraryA));
s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back(
FunctionHook("LoadLibraryW", NULL, &Hooked_LoadLibraryW));
s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back(
FunctionHook("LoadLibraryExA", NULL, &Hooked_LoadLibraryExA));
s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back(
FunctionHook("LoadLibraryExW", NULL, &Hooked_LoadLibraryExW));
s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back(
FunctionHook("GetProcAddress", NULL, &Hooked_GetProcAddress));
for(const char *apiset :
{"api-ms-win-core-libraryloader-l1-1-0.dll", "api-ms-win-core-libraryloader-l1-1-1.dll",
"api-ms-win-core-libraryloader-l1-1-2.dll", "api-ms-win-core-libraryloader-l1-2-0.dll",
"api-ms-win-core-libraryloader-l1-2-1.dll"})
{
s_HookData->DllHooks[apiset].FunctionHooks.push_back(
FunctionHook("LoadLibraryA", NULL, &Hooked_LoadLibraryA));
s_HookData->DllHooks[apiset].FunctionHooks.push_back(
FunctionHook("LoadLibraryW", NULL, &Hooked_LoadLibraryW));
s_HookData->DllHooks[apiset].FunctionHooks.push_back(
FunctionHook("LoadLibraryExA", NULL, &Hooked_LoadLibraryExA));
s_HookData->DllHooks[apiset].FunctionHooks.push_back(
FunctionHook("LoadLibraryExW", NULL, &Hooked_LoadLibraryExW));
s_HookData->DllHooks[apiset].FunctionHooks.push_back(
FunctionHook("GetProcAddress", NULL, &Hooked_GetProcAddress));
}
GetModuleHandleEx(
GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
(LPCTSTR)&s_HookData, &s_HookData->ownmodule);
InitHookData();
}
// hook all functions for currently loaded modules.
@@ -952,6 +978,30 @@ bool LibraryHooks::Detect(const char *identifier)
return ret;
}
void Win32_RegisterManualModuleHooking()
{
InitHookData();
s_HookData->hookAll = false;
}
void Win32_InterceptLibraryLoads(std::function<HMODULE(const rdcstr &, HANDLE, DWORD)> callback)
{
s_HookData->libraryIntercept = callback;
}
void Win32_ManualHookModule(rdcstr modName, HMODULE module)
{
for(auto it = s_HookData->DllHooks.begin(); it != s_HookData->DllHooks.end(); ++it)
std::sort(it->second.FunctionHooks.begin(), it->second.FunctionHooks.end());
modName = strlower(modName);
s_HookData->DllHooks[modName].module = module;
s_HookData->ApplyHooks(modName.c_str(), module);
}
// android only hooking functions, not used on win32
ScopedSuppressHooking::ScopedSuppressHooking()
{