diff --git a/renderdoc/driver/d3d12/d3d12_common.h b/renderdoc/driver/d3d12/d3d12_common.h index 0258f8e0c..cf04d8192 100644 --- a/renderdoc/driver/d3d12/d3d12_common.h +++ b/renderdoc/driver/d3d12/d3d12_common.h @@ -59,6 +59,9 @@ struct D3D12MarkerRegion bool EnableD3D12DebugLayer(PFN_D3D12_GET_DEBUG_INTERFACE getDebugInterface = NULL); HRESULT EnumAdapterByLuid(IDXGIFactory1 *factory, LUID luid, IDXGIAdapter **pAdapter); +void D3D12_PrepareReplaySDKVersion(UINT SDKVersion, bytebuf d3d12core, HMODULE d3d12lib); +void D3D12_CleanupReplaySDK(); + inline void SetObjName(ID3D12Object *obj, const rdcstr &utf8name) { obj->SetName(StringFormat::UTF82Wide(utf8name).c_str()); diff --git a/renderdoc/driver/d3d12/d3d12_replay.cpp b/renderdoc/driver/d3d12/d3d12_replay.cpp index d5b239390..efe1aa8b8 100644 --- a/renderdoc/driver/d3d12/d3d12_replay.cpp +++ b/renderdoc/driver/d3d12/d3d12_replay.cpp @@ -74,6 +74,7 @@ void D3D12Replay::Shutdown() // the this pointer is free'd after this point FreeLibrary(D3D12Lib); + D3D12_CleanupReplaySDK(); } void D3D12Replay::Initialise(IDXGIFactory1 *factory) @@ -3921,6 +3922,7 @@ ReplayStatus D3D12_CreateReplayDevice(RDCFile *rdc, const ReplayOptions &opts, I } D3D12InitParams initParams; + bytebuf D3D12Core; uint64_t ver = D3D12InitParams::CurrentVersion; @@ -3968,6 +3970,28 @@ ReplayStatus D3D12_CreateReplayDevice(RDCFile *rdc, const ReplayOptions &opts, I ToStr(GPUVendorFromPCIVendor(initParams.AdapterDesc.VendorId)).c_str(), initParams.AdapterDesc.Description); + sectionIdx = rdc->SectionIndex(SectionType::D3D12Core); + + if(sectionIdx >= 0) + { + SectionProperties props = rdc->GetSectionProperties(sectionIdx); + reader = rdc->ReadSection(sectionIdx); + + D3D12Core.resize(props.uncompressedSize); + reader->Read(D3D12Core.data(), props.uncompressedSize); + + RDCASSERT(reader->AtEnd()); + delete reader; + } + } + + if(initParams.MinimumFeatureLevel < D3D_FEATURE_LEVEL_11_0) + initParams.MinimumFeatureLevel = D3D_FEATURE_LEVEL_11_0; + + D3D12_PrepareReplaySDKVersion(initParams.SDKVersion, D3D12Core, D3D12Lib); + + if(rdc) + { using PFN_ENABLE_EXPERIMENTAL = decltype(&D3D12EnableExperimentalFeatures); PFN_ENABLE_EXPERIMENTAL EnableExperimental = @@ -3987,9 +4011,6 @@ ReplayStatus D3D12_CreateReplayDevice(RDCFile *rdc, const ReplayOptions &opts, I } } - if(initParams.MinimumFeatureLevel < D3D_FEATURE_LEVEL_11_0) - initParams.MinimumFeatureLevel = D3D_FEATURE_LEVEL_11_0; - const bool isProxy = (rdc == NULL); AMDRGPControl *rgp = NULL; diff --git a/renderdoc/driver/d3d12/d3d12_sdk_select.cpp b/renderdoc/driver/d3d12/d3d12_sdk_select.cpp new file mode 100644 index 000000000..c90332108 --- /dev/null +++ b/renderdoc/driver/d3d12/d3d12_sdk_select.cpp @@ -0,0 +1,271 @@ +/****************************************************************************** + * The MIT License (MIT) + * + * Copyright (c) 2021 Baldur Karlsson + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + ******************************************************************************/ + +#include "core/settings.h" +#include "hooks/hooks.h" +#include "strings/string_utils.h" +#include "d3d12_common.h" + +RDOC_CONFIG(rdcstr, D3D12_D3D12CorePath, "", + "The location of the D3D12Core library. This path should be the directory that " + "contains the D3D12Core.dll that you want to use."); + +// special hooking functions exposed on windows for hooking while in replay mode +void Win32_RegisterManualModuleHooking(); +void Win32_InterceptLibraryLoads(std::function callback); +void Win32_ManualHookModule(rdcstr modName, HMODULE module); + +namespace +{ +DWORD SystemCoreVersion; +rdcstr D3D12Core_Override_Path; +rdcstr D3D12Core_Temp_Path; +}; + +MIDL_INTERFACE("DFAFDD2C-355F-4CB3-A8B2-EA7F9260148B") +ID3D12CoreModule : public IUnknown +{ +public: + virtual DWORD STDMETHODCALLTYPE LOEnter(void) = 0; + virtual DWORD STDMETHODCALLTYPE LOLeave(void) = 0; + virtual DWORD STDMETHODCALLTYPE LOTryEnter(void) = 0; + virtual HRESULT STDMETHODCALLTYPE Initialize(DWORD, LPCSTR) = 0; + virtual DWORD STDMETHODCALLTYPE GetSDKVersion(void) = 0; + virtual HRESULT STDMETHODCALLTYPE GetDllExports(void *) = 0; +}; + +struct WrappedCoreModule : public ID3D12CoreModule +{ + unsigned int m_iRefcount = 0; + ID3D12CoreModule *m_pReal = NULL; + + WrappedCoreModule(ID3D12CoreModule *real) : m_iRefcount(1), m_pReal(real) {} + ~WrappedCoreModule() { SAFE_RELEASE(m_pReal); } + ////////////////////////////// + // implement IUnknown + HRESULT STDMETHODCALLTYPE QueryInterface(REFIID riid, void **ppvObject) { return E_NOINTERFACE; } + ULONG STDMETHODCALLTYPE AddRef() + { + InterlockedIncrement(&m_iRefcount); + return m_iRefcount; + } + ULONG STDMETHODCALLTYPE Release() + { + unsigned int ret = InterlockedDecrement(&m_iRefcount); + if(ret == 0) + delete this; + return ret; + } + ////////////////////////////// + // implement ID3D12CoreModule + virtual DWORD STDMETHODCALLTYPE LOEnter(void) { return m_pReal->LOEnter(); } + virtual DWORD STDMETHODCALLTYPE LOLeave(void) { return m_pReal->LOLeave(); } + virtual DWORD STDMETHODCALLTYPE LOTryEnter(void) { return m_pReal->LOTryEnter(); } + virtual HRESULT STDMETHODCALLTYPE Initialize(DWORD version, LPCSTR unknown) + { + // D3D12 should always think it's loading the system core version + RDCASSERT(version == SystemCoreVersion); + + // initialize the actual core (which might in some cases be the system version) with its own + // version so it doesn't complain + return m_pReal->Initialize(m_pReal->GetSDKVersion(), unknown); + } + + virtual DWORD STDMETHODCALLTYPE GetSDKVersion(void) { return SystemCoreVersion; } + virtual HRESULT STDMETHODCALLTYPE GetDllExports(void *a) { return m_pReal->GetDllExports(a); } +}; + +using PFN_D3D12_GET_INTERFACE = decltype(&D3D12GetInterface); +HookedFunction D3D12GetInterface_hook; + +HMODULE Hooked_D3D12LoadLibrary(const rdcstr &filename, HANDLE h, DWORD flags) +{ + // if we detect the intercepted call to D3D12Core.dll and we have a redirect path, load that + // redirect path instead + if(strlower(filename).contains("d3d12core.dll") && !D3D12Core_Override_Path.empty()) + return LoadLibraryExW(StringFormat::UTF82Wide(D3D12Core_Override_Path).c_str(), NULL, flags); + + return NULL; +} + +HRESULT WINAPI Hooked_D3D12GetInterface(_In_ REFCLSID rclsid, _In_ REFIID riid, + _COM_Outptr_opt_ void **ppvDebug) +{ + HRESULT ret = D3D12GetInterface_hook()(rclsid, riid, ppvDebug); + + // intercept the interface with our own wrapper to ensure version checking is silenced + if(SUCCEEDED(ret) && riid == __uuidof(ID3D12CoreModule)) + *ppvDebug = (ID3D12CoreModule *)(new WrappedCoreModule((ID3D12CoreModule *)*ppvDebug)); + + return ret; +} + +void D3D12_PrepareReplaySDKVersion(UINT SDKVersion, bytebuf d3d12core_file, HMODULE d3d12lib) +{ + // D3D12Core shouldn't be loaded at this point, but it might be due to bugs. If it is, we can't do + // anything to change it anymore so we have to just handle what we have + HMODULE D3D12Core = GetModuleHandleA("D3D12Core.dll"); + if(D3D12Core != NULL) + { + DWORD *ver_ptr = (DWORD *)GetProcAddress(D3D12Core, "D3D12SDKVersion"); + + // if the core that's loaded is sufficient, don't show any warnings + if(ver_ptr && SDKVersion <= *ver_ptr) + return; + + RDCWARN( + "D3D12Core.dll was already loaded before replay started. This may be caused by a D3D12 " + "runtime bug if the validation layers are enabled, that means D3D12 is never unloaded."); + + if(ver_ptr) + RDCWARN("The existing D3D12Core.dll is version %u but this capture requires version %u", + *ver_ptr, SDKVersion); + else + RDCWARN("The existing D3D12Core.dll had an unknown version, this capture requires version %u", + SDKVersion); + + return; + } + + static bool core_fetched = false, hooks_applied = false; + + // if we don't have the system core version yet + if(!core_fetched) + { + core_fetched = true; + + // default to 0 + SystemCoreVersion = 0; + + // get the system path to load it explicitly + wchar_t sys_path[MAX_PATH + 16] = {}; + GetSystemDirectoryW(sys_path, MAX_PATH); + + wcscat_s(sys_path, L"\\D3D12Core.dll"); + + HMODULE real_sys = LoadLibraryW(sys_path); + if(real_sys) + { + DWORD *ver_ptr = (DWORD *)GetProcAddress(real_sys, "D3D12SDKVersion"); + + if(ver_ptr) + SystemCoreVersion = *ver_ptr; + else + RDCERR("D3D12Core.dll loaded from %ls doesn't have D3D12SDKVersion export!", sys_path); + + FreeLibrary(real_sys); + + RDCLOG("System D3D12 runtime is version %u", SystemCoreVersion); + } + else + { + RDCLOG("No system D3D12 runtime found at %ls.", sys_path); + + // if the captured SDK version was greater than 1 then most likely this capture will fail to + // replay, but we can still try to replay (there's no guarantee a user actually used features + // exclusive to the new runtime). + if(SDKVersion > 1) + RDCWARN( + "Capture was made with runtime version %u but this system does not support D3D12 " + "runtimes, possible incompatibility"); + } + } + + // if the system doesn't have a core DLL we can't intercept and point to our own runtime, so just + // abort here before doing anything potentially dangerous below. + if(SystemCoreVersion == 0) + return; + + // similarly, if the system version is enough then the user didn't use a new runtime (or they used + // what was at the time a new runtime but is now available in the system...), so also abort. + // That means we'll only do the interception & patching when we think it's really needed. + if(SDKVersion <= SystemCoreVersion) + return; + + // finally we're at a point where we will hook to force the library we want. + + if(!hooks_applied) + { + Win32_RegisterManualModuleHooking(); + + D3D12GetInterface_hook.Register("d3d12core.dll", "D3D12GetInterface", Hooked_D3D12GetInterface); + + Win32_InterceptLibraryLoads(Hooked_D3D12LoadLibrary); + } + + // we do this always, even if the hooks are already applied, because this module has possibly been + // reloaded and needs to be re-hooked each time + Win32_ManualHookModule("d3d12.dll", d3d12lib); + + // *always* use the user's path if it exists + D3D12Core_Override_Path = D3D12_D3D12CorePath(); + + if(D3D12Core_Override_Path.empty() || !FileIO::exists(D3D12Core_Override_Path.c_str())) + { + if(d3d12core_file.empty()) + { + RDCERR( + "No D3D12Core.dll embedded in capture but we need a newer one (version %u) to properly " + "replay this capture", + SDKVersion); + return; + } + + // find an appropriate spot to write this file. Other instances of RenderDoc might be running so + // we try a few different variants + for(uint32_t i = 0; i < 32; i++) + { + rdcstr filename = StringFormat::Fmt("%s/RenderDoc/D3D12Core/%u.ver%u/D3D12Core.dll", + FileIO::GetTempFolderFilename().c_str(), i, SDKVersion); + + FileIO::CreateParentDirectory(filename); + + FILE *f = FileIO::fopen(filename.c_str(), FileIO::WriteBinary); + + // if we can write to this file, we have exclusive use of it so let's write it and use it + if(f) + { + FileIO::fwrite(d3d12core_file.data(), 1, d3d12core_file.size(), f); + FileIO::fclose(f); + + D3D12Core_Override_Path = filename; + D3D12Core_Temp_Path = filename; + break; + } + } + + if(D3D12Core_Override_Path.empty() || !FileIO::exists(D3D12Core_Override_Path.c_str())) + { + RDCERR("Couldn't write embedded D3D12Core.dll to disk! system dll will be used"); + } + } +} + +void D3D12_CleanupReplaySDK() +{ + if(!D3D12Core_Temp_Path.empty() && FileIO::exists(D3D12Core_Temp_Path.c_str())) + { + FileIO::Delete(D3D12Core_Temp_Path.c_str()); + } +} diff --git a/renderdoc/driver/d3d12/renderdoc_d3d12.vcxproj b/renderdoc/driver/d3d12/renderdoc_d3d12.vcxproj index 47af541ae..366f7920b 100644 --- a/renderdoc/driver/d3d12/renderdoc_d3d12.vcxproj +++ b/renderdoc/driver/d3d12/renderdoc_d3d12.vcxproj @@ -136,6 +136,7 @@ + diff --git a/renderdoc/driver/d3d12/renderdoc_d3d12.vcxproj.filters b/renderdoc/driver/d3d12/renderdoc_d3d12.vcxproj.filters index bbabb65a7..ad5f6f928 100644 --- a/renderdoc/driver/d3d12/renderdoc_d3d12.vcxproj.filters +++ b/renderdoc/driver/d3d12/renderdoc_d3d12.vcxproj.filters @@ -201,5 +201,8 @@ Wrapped + + Util + \ No newline at end of file diff --git a/renderdoc/os/win32/win32_hook.cpp b/renderdoc/os/win32/win32_hook.cpp index 59cc38bd1..b0ce83013 100644 --- a/renderdoc/os/win32/win32_hook.cpp +++ b/renderdoc/os/win32/win32_hook.cpp @@ -148,21 +148,17 @@ struct DllHookset struct CachedHookData { - CachedHookData() - { - ownmodule = NULL; - missedOrdinals = false; - RDCEraseEl(lowername); - } + bool hookAll = true; std::map DllHooks; - HMODULE ownmodule; + HMODULE ownmodule = NULL; Threading::CriticalSection lock; - char lowername[512]; + char lowername[512] = {}; std::set ignores; - bool missedOrdinals; + bool missedOrdinals = false; + std::function libraryIntercept; int32_t posthooking = 0; @@ -564,6 +560,9 @@ static void ForAllModules(std::function callbac static void HookAllModules() { + if(!s_HookData->hookAll) + return; + ForAllModules( [](const MODULEENTRY32 &me32) { s_HookData->ApplyHooks(me32.szModule, me32.hModule); }); @@ -636,6 +635,15 @@ static bool IsAPISet(const char *filename) HMODULE WINAPI Hooked_LoadLibraryExA(LPCSTR lpLibFileName, HANDLE fileHandle, DWORD flags) { bool dohook = true; + + if(s_HookData->libraryIntercept) + { + HMODULE ret = s_HookData->libraryIntercept(lpLibFileName, fileHandle, flags); + if(ret) + return ret; + dohook = false; + } + if(flags == 0 && GetModuleHandleA(lpLibFileName)) dohook = false; @@ -662,6 +670,16 @@ HMODULE WINAPI Hooked_LoadLibraryExA(LPCSTR lpLibFileName, HANDLE fileHandle, DW HMODULE WINAPI Hooked_LoadLibraryExW(LPCWSTR lpLibFileName, HANDLE fileHandle, DWORD flags) { bool dohook = true; + + if(s_HookData->libraryIntercept) + { + HMODULE ret = + s_HookData->libraryIntercept(StringFormat::Wide2UTF8(lpLibFileName), fileHandle, flags); + if(ret) + return ret; + dohook = false; + } + if(flags == 0 && GetModuleHandleW(lpLibFileName)) dohook = false; @@ -814,6 +832,46 @@ FARPROC WINAPI Hooked_GetProcAddress(HMODULE mod, LPCSTR func) return GetProcAddress(mod, func); } +static void InitHookData() +{ + if(!s_HookData) + { + s_HookData = new CachedHookData; + + RDCASSERT(s_HookData->DllHooks.empty()); + s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back( + FunctionHook("LoadLibraryA", NULL, &Hooked_LoadLibraryA)); + s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back( + FunctionHook("LoadLibraryW", NULL, &Hooked_LoadLibraryW)); + s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back( + FunctionHook("LoadLibraryExA", NULL, &Hooked_LoadLibraryExA)); + s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back( + FunctionHook("LoadLibraryExW", NULL, &Hooked_LoadLibraryExW)); + s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back( + FunctionHook("GetProcAddress", NULL, &Hooked_GetProcAddress)); + + for(const char *apiset : + {"api-ms-win-core-libraryloader-l1-1-0.dll", "api-ms-win-core-libraryloader-l1-1-1.dll", + "api-ms-win-core-libraryloader-l1-1-2.dll", "api-ms-win-core-libraryloader-l1-2-0.dll", + "api-ms-win-core-libraryloader-l1-2-1.dll"}) + { + s_HookData->DllHooks[apiset].FunctionHooks.push_back( + FunctionHook("LoadLibraryA", NULL, &Hooked_LoadLibraryA)); + s_HookData->DllHooks[apiset].FunctionHooks.push_back( + FunctionHook("LoadLibraryW", NULL, &Hooked_LoadLibraryW)); + s_HookData->DllHooks[apiset].FunctionHooks.push_back( + FunctionHook("LoadLibraryExA", NULL, &Hooked_LoadLibraryExA)); + s_HookData->DllHooks[apiset].FunctionHooks.push_back( + FunctionHook("LoadLibraryExW", NULL, &Hooked_LoadLibraryExW)); + s_HookData->DllHooks[apiset].FunctionHooks.push_back( + FunctionHook("GetProcAddress", NULL, &Hooked_GetProcAddress)); + } + + GetModuleHandleEx( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCTSTR)&s_HookData, &s_HookData->ownmodule); + } +} void LibraryHooks::RegisterFunctionHook(const char *libraryName, const FunctionHook &hook) { @@ -847,39 +905,7 @@ void LibraryHooks::IgnoreLibrary(const char *libraryName) void LibraryHooks::BeginHookRegistration() { - s_HookData = new CachedHookData; - RDCASSERT(s_HookData->DllHooks.empty()); - s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back( - FunctionHook("LoadLibraryA", NULL, &Hooked_LoadLibraryA)); - s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back( - FunctionHook("LoadLibraryW", NULL, &Hooked_LoadLibraryW)); - s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back( - FunctionHook("LoadLibraryExA", NULL, &Hooked_LoadLibraryExA)); - s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back( - FunctionHook("LoadLibraryExW", NULL, &Hooked_LoadLibraryExW)); - s_HookData->DllHooks["kernel32.dll"].FunctionHooks.push_back( - FunctionHook("GetProcAddress", NULL, &Hooked_GetProcAddress)); - - for(const char *apiset : - {"api-ms-win-core-libraryloader-l1-1-0.dll", "api-ms-win-core-libraryloader-l1-1-1.dll", - "api-ms-win-core-libraryloader-l1-1-2.dll", "api-ms-win-core-libraryloader-l1-2-0.dll", - "api-ms-win-core-libraryloader-l1-2-1.dll"}) - { - s_HookData->DllHooks[apiset].FunctionHooks.push_back( - FunctionHook("LoadLibraryA", NULL, &Hooked_LoadLibraryA)); - s_HookData->DllHooks[apiset].FunctionHooks.push_back( - FunctionHook("LoadLibraryW", NULL, &Hooked_LoadLibraryW)); - s_HookData->DllHooks[apiset].FunctionHooks.push_back( - FunctionHook("LoadLibraryExA", NULL, &Hooked_LoadLibraryExA)); - s_HookData->DllHooks[apiset].FunctionHooks.push_back( - FunctionHook("LoadLibraryExW", NULL, &Hooked_LoadLibraryExW)); - s_HookData->DllHooks[apiset].FunctionHooks.push_back( - FunctionHook("GetProcAddress", NULL, &Hooked_GetProcAddress)); - } - - GetModuleHandleEx( - GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, - (LPCTSTR)&s_HookData, &s_HookData->ownmodule); + InitHookData(); } // hook all functions for currently loaded modules. @@ -952,6 +978,30 @@ bool LibraryHooks::Detect(const char *identifier) return ret; } +void Win32_RegisterManualModuleHooking() +{ + InitHookData(); + + s_HookData->hookAll = false; +} + +void Win32_InterceptLibraryLoads(std::function callback) +{ + s_HookData->libraryIntercept = callback; +} + +void Win32_ManualHookModule(rdcstr modName, HMODULE module) +{ + for(auto it = s_HookData->DllHooks.begin(); it != s_HookData->DllHooks.end(); ++it) + std::sort(it->second.FunctionHooks.begin(), it->second.FunctionHooks.end()); + + modName = strlower(modName); + + s_HookData->DllHooks[modName].module = module; + + s_HookData->ApplyHooks(modName.c_str(), module); +} + // android only hooking functions, not used on win32 ScopedSuppressHooking::ScopedSuppressHooking() {