From 75623b448732274781ec19d9fd60a51753c14bf2 Mon Sep 17 00:00:00 2001 From: baldurk Date: Tue, 26 Jun 2018 12:14:46 +0100 Subject: [PATCH] Clean up D3D/DXGI hooks. Reduce duplication and separate replay code * Previously we were going through the hooks for device/factory creation, which did barely anything actually useful. Instead we can call directly to the real function and wrap the result manually. --- renderdoc/driver/d3d11/d3d11_device.cpp | 5 +- renderdoc/driver/d3d11/d3d11_device.h | 2 +- renderdoc/driver/d3d11/d3d11_hooks.cpp | 350 +++++++----------------- renderdoc/driver/d3d11/d3d11_replay.cpp | 30 +- renderdoc/driver/d3d12/d3d12_common.cpp | 50 ++++ renderdoc/driver/d3d12/d3d12_common.h | 2 + renderdoc/driver/d3d12/d3d12_device.cpp | 5 +- renderdoc/driver/d3d12/d3d12_device.h | 2 +- renderdoc/driver/d3d12/d3d12_hooks.cpp | 87 +----- renderdoc/driver/d3d12/d3d12_replay.cpp | 33 ++- renderdoc/driver/dxgi/dxgi_hooks.cpp | 87 +----- 11 files changed, 206 insertions(+), 447 deletions(-) diff --git a/renderdoc/driver/d3d11/d3d11_device.cpp b/renderdoc/driver/d3d11/d3d11_device.cpp index 820500049..c463b779a 100644 --- a/renderdoc/driver/d3d11/d3d11_device.cpp +++ b/renderdoc/driver/d3d11/d3d11_device.cpp @@ -53,7 +53,7 @@ void WrappedID3D11Device::NewSwapchainBuffer(IUnknown *backbuffer) } } -WrappedID3D11Device::WrappedID3D11Device(ID3D11Device *realDevice, D3D11InitParams *params) +WrappedID3D11Device::WrappedID3D11Device(ID3D11Device *realDevice, D3D11InitParams params) : m_RefCounter(realDevice, false), m_SoftRefCounter(NULL, false), m_pDevice(realDevice), @@ -202,8 +202,7 @@ WrappedID3D11Device::WrappedID3D11Device(ID3D11Device *realDevice, D3D11InitPara m_Replay.SetDevice(this); - if(params) - m_InitParams = *params; + m_InitParams = params; if(realDevice) m_DebugManager = new D3D11DebugManager(this); diff --git a/renderdoc/driver/d3d11/d3d11_device.h b/renderdoc/driver/d3d11/d3d11_device.h index 063054dab..454271fa6 100644 --- a/renderdoc/driver/d3d11/d3d11_device.h +++ b/renderdoc/driver/d3d11/d3d11_device.h @@ -410,7 +410,7 @@ public: static const int AllocPoolCount = 4; ALLOCATE_WITH_WRAPPED_POOL(WrappedID3D11Device, AllocPoolCount); - WrappedID3D11Device(ID3D11Device *realDevice, D3D11InitParams *params); + WrappedID3D11Device(ID3D11Device *realDevice, D3D11InitParams params); void SetInitParams(const D3D11InitParams ¶ms, uint64_t sectionVersion) { m_InitParams = params; diff --git a/renderdoc/driver/d3d11/d3d11_hooks.cpp b/renderdoc/driver/d3d11/d3d11_hooks.cpp index b033ee8d4..3da09c9f0 100644 --- a/renderdoc/driver/d3d11/d3d11_hooks.cpp +++ b/renderdoc/driver/d3d11/d3d11_hooks.cpp @@ -59,6 +59,14 @@ typedef HRESULT (*PFNNVCreateDeviceAndSwapChain)( _COM_Outptr_opt_ ID3D11Device **, _Out_opt_ D3D_FEATURE_LEVEL *, _COM_Outptr_opt_ ID3D11DeviceContext **, NVAPI_DEVICE_FEATURE_LEVEL *); +// this is the type of the lambda we use to route the call out to the 'real' function inside our +// generic wrapper. +// Could be any of D3D11CreateDevice, D3D11CreateDeviceAndSwapChain, or the nvapi equivalents +typedef std::function + RealFunction; + #define NVENCAPI __stdcall enum NVENCSTATUS { @@ -162,18 +170,6 @@ public: } void OptionsUpdated(const char *libName) {} - static HRESULT CreateWrappedDeviceAndSwapChain( - __in_opt IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, - __in_ecount_opt(FeatureLevels) CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, - UINT SDKVersion, __in_opt CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, - __out_opt IDXGISwapChain **ppSwapChain, __out_opt ID3D11Device **ppDevice, - __out_opt D3D_FEATURE_LEVEL *pFeatureLevel, __out_opt ID3D11DeviceContext **ppImmediateContext) - { - return d3d11hooks.Create_Internal(pAdapter, DriverType, Software, Flags, pFeatureLevels, - FeatureLevels, SDKVersion, pSwapChainDesc, ppSwapChain, - ppDevice, pFeatureLevel, ppImmediateContext); - } - private: static D3D11Hook d3d11hooks; @@ -190,8 +186,8 @@ private: // re-entrancy detection (can happen in rare cases with e.g. fraps) bool m_InsideCreate; - HRESULT Create_Internal(__in_opt IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, - HMODULE Software, UINT Flags, + HRESULT Create_Internal(RealFunction real, __in_opt IDXGIAdapter *pAdapter, + D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, __in_ecount_opt(FeatureLevels) CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion, __in_opt CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, @@ -199,57 +195,24 @@ private: __out_opt D3D_FEATURE_LEVEL *pFeatureLevel, __out_opt ID3D11DeviceContext **ppImmediateContext) { - // if we're already inside a wrapped create i.e. this function, then DON'T do anything - // special. Just grab the trampolined function and call it. + // if we're already inside a wrapped create, then DON'T do anything special. Just call onwards if(m_InsideCreate) { - PFN_D3D11_CREATE_DEVICE_AND_SWAP_CHAIN createFunc = NULL; - - // shouldn't ever get in here if we're in the case without hooks but let's be safe. - if(m_HasHooks) - { - createFunc = CreateDeviceAndSwapChain(); - } - else - { - HMODULE d3d11 = GetModuleHandleA("d3d11.dll"); - - if(d3d11) - { - createFunc = (PFN_D3D11_CREATE_DEVICE_AND_SWAP_CHAIN)GetProcAddress( - d3d11, "D3D11CreateDeviceAndSwapChain"); - } - else - { - RDCERR("Something went seriously wrong, d3d11.dll couldn't be loaded!"); - return E_UNEXPECTED; - } - } - - return createFunc(pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, - SDKVersion, pSwapChainDesc, ppSwapChain, ppDevice, pFeatureLevel, - ppImmediateContext); + return real(pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, + pSwapChainDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext); } m_InsideCreate = true; RDCDEBUG("Call to Create_Internal Flags %x", Flags); - bool reading = RenderDoc::Inst().IsReplayApp(); + // we should no longer go through here in the replay application + RDCASSERT(!RenderDoc::Inst().IsReplayApp()); - if(reading) - { - RDCDEBUG("In replay app"); - } - - if(!reading && RenderDoc::Inst().GetCaptureOptions().apiValidation) - { + if(RenderDoc::Inst().GetCaptureOptions().apiValidation) Flags |= D3D11_CREATE_DEVICE_DEBUG; - } else - { Flags &= ~D3D11_CREATE_DEVICE_DEBUG; - } DXGI_SWAP_CHAIN_DESC swapDesc; DXGI_SWAP_CHAIN_DESC *pUsedSwapDesc = NULL; @@ -267,22 +230,6 @@ private: RDCDEBUG("Calling real createdevice..."); - PFN_D3D11_CREATE_DEVICE_AND_SWAP_CHAIN createFunc = - (PFN_D3D11_CREATE_DEVICE_AND_SWAP_CHAIN)GetProcAddress(GetModuleHandleA("d3d11.dll"), - "D3D11CreateDeviceAndSwapChain"); - - // shouldn't ever get here, we should either have it from procaddress or the trampoline, but - // let's be - // safe. - if(createFunc == NULL) - { - RDCERR("Something went seriously wrong with the hooks!"); - - m_InsideCreate = false; - - return E_UNEXPECTED; - } - // Hack for D3DGear which crashes if ppDevice is NULL ID3D11Device *dummydev = NULL; bool dummyUsed = false; @@ -293,8 +240,8 @@ private: } HRESULT ret = - createFunc(pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, - pUsedSwapDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext); + real(pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, + pUsedSwapDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext); SAFE_RELEASE(dummydev); if(dummyUsed) @@ -306,14 +253,41 @@ private: suppress = (Flags & D3D11_CREATE_DEVICE_PREVENT_ALTERING_LAYER_SETTINGS_FROM_REGISTRY) != 0; - if(suppress && !reading) + if(suppress) { RDCLOG("Application requested not to be hooked."); } else if(SUCCEEDED(ret) && ppDevice) { - WrapDevice(ppDevice, DriverType, Flags, SDKVersion, FeatureLevels, pFeatureLevels, - ppImmediateContext, ppSwapChain, pSwapChainDesc); + RDCDEBUG("succeeded and hooking."); + + if(!WrappedID3D11Device::IsAlloc(*ppDevice)) + { + D3D11InitParams params; + params.DriverType = DriverType; + params.Flags = Flags; + params.SDKVersion = SDKVersion; + params.NumFeatureLevels = FeatureLevels; + if(FeatureLevels > 0) + memcpy(params.FeatureLevels, pFeatureLevels, sizeof(D3D_FEATURE_LEVEL) * FeatureLevels); + + WrappedID3D11Device *wrap = new WrappedID3D11Device(*ppDevice, params); + + RDCDEBUG("created wrapped device."); + + *ppDevice = wrap; + + if(ppImmediateContext) + { + if(*ppImmediateContext) + (*ppImmediateContext)->Release(); + wrap->GetImmediateContext(ppImmediateContext); + } + + if(ppSwapChain && *ppSwapChain) + *ppSwapChain = new WrappedIDXGISwapChain4( + *ppSwapChain, pSwapChainDesc ? pSwapChainDesc->OutputWindow : NULL, wrap); + } } else if(SUCCEEDED(ret)) { @@ -329,53 +303,16 @@ private: return ret; } - void WrapDevice(ID3D11Device **ppDevice, D3D_DRIVER_TYPE DriverType, UINT Flags, UINT SDKVersion, - UINT FeatureLevels, CONST D3D_FEATURE_LEVEL *pFeatureLevels, - ID3D11DeviceContext **ppImmediateContext, IDXGISwapChain **ppSwapChain, - CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc) - { - RDCDEBUG("succeeded and hooking."); - - if(!WrappedID3D11Device::IsAlloc(*ppDevice)) - { - D3D11InitParams params; - params.DriverType = DriverType; - params.Flags = Flags; - params.SDKVersion = SDKVersion; - params.NumFeatureLevels = FeatureLevels; - if(FeatureLevels > 0) - memcpy(params.FeatureLevels, pFeatureLevels, sizeof(D3D_FEATURE_LEVEL) * FeatureLevels); - - WrappedID3D11Device *wrap = new WrappedID3D11Device(*ppDevice, ¶ms); - - RDCDEBUG("created wrapped device."); - - *ppDevice = wrap; - - if(ppImmediateContext) - { - if(*ppImmediateContext) - (*ppImmediateContext)->Release(); - wrap->GetImmediateContext(ppImmediateContext); - } - - if(ppSwapChain && *ppSwapChain) - *ppSwapChain = new WrappedIDXGISwapChain4( - *ppSwapChain, pSwapChainDesc ? pSwapChainDesc->OutputWindow : NULL, wrap); - } - } - static HRESULT WINAPI D3D11CreateDevice_hook( __in_opt IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, __in_ecount_opt(FeatureLevels) CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion, __out_opt ID3D11Device **ppDevice, __out_opt D3D_FEATURE_LEVEL *pFeatureLevel, __out_opt ID3D11DeviceContext **ppImmediateContext) { - HRESULT ret = d3d11hooks.Create_Internal(pAdapter, DriverType, Software, Flags, pFeatureLevels, - FeatureLevels, SDKVersion, NULL, NULL, ppDevice, - pFeatureLevel, ppImmediateContext); - - return ret; + // just forward the call with NULL swapchain parameters + return D3D11CreateDeviceAndSwapChain_hook(pAdapter, DriverType, Software, Flags, pFeatureLevels, + FeatureLevels, SDKVersion, NULL, NULL, ppDevice, + pFeatureLevel, ppImmediateContext); } static HRESULT WINAPI D3D11CreateDeviceAndSwapChain_hook( @@ -385,9 +322,26 @@ private: __out_opt IDXGISwapChain **ppSwapChain, __out_opt ID3D11Device **ppDevice, __out_opt D3D_FEATURE_LEVEL *pFeatureLevel, __out_opt ID3D11DeviceContext **ppImmediateContext) { - HRESULT ret = d3d11hooks.Create_Internal(pAdapter, DriverType, Software, Flags, pFeatureLevels, - FeatureLevels, SDKVersion, pSwapChainDesc, ppSwapChain, - ppDevice, pFeatureLevel, ppImmediateContext); + PFN_D3D11_CREATE_DEVICE_AND_SWAP_CHAIN createFunc = d3d11hooks.CreateDeviceAndSwapChain(); + + if(createFunc == NULL) + { + RDCWARN("Call to D3D11CreateDeviceAndSwapChain_hook without onward function pointer"); + createFunc = (PFN_D3D11_CREATE_DEVICE_AND_SWAP_CHAIN)GetProcAddress( + GetModuleHandleA("d3d11.dll"), "D3D11CreateDeviceAndSwapChain"); + } + + // shouldn't ever get here, we should either have it from procaddress or the hook function, but + // let's be safe. + if(createFunc == NULL) + { + RDCERR("Something went seriously wrong with the hooks!"); + return E_UNEXPECTED; + } + + HRESULT ret = d3d11hooks.Create_Internal( + createFunc, pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, + SDKVersion, pSwapChainDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext); return ret; } @@ -413,55 +367,21 @@ private: ID3D11DeviceContext **ppImmediateContext, NVAPI_DEVICE_FEATURE_LEVEL *outNVLevel) { - RDCDEBUG("Call to nvapi_CreateDevice"); - - bool reading = RenderDoc::Inst().IsReplayApp(); - - if(reading) - { - RDCDEBUG("In replay app"); - } - - if(!reading && RenderDoc::Inst().GetCaptureOptions().apiValidation) - { - Flags |= D3D11_CREATE_DEVICE_DEBUG; - } - else - { - Flags &= ~D3D11_CREATE_DEVICE_DEBUG; - } - - RDCDEBUG("Calling real createdevice..."); - - HRESULT ret = d3d11hooks.nvapi_CreateDevice_real( - pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, ppDevice, - pFeatureLevel, ppImmediateContext, outNVLevel); - - RDCDEBUG("Called real createdevice..."); - - bool suppress = false; - - suppress = (Flags & D3D11_CREATE_DEVICE_PREVENT_ALTERING_LAYER_SETTINGS_FROM_REGISTRY) != 0; - - if(suppress && !reading) - { - RDCDEBUG("Application requested not to be hooked."); - } - else if(SUCCEEDED(ret) && ppDevice) - { - d3d11hooks.WrapDevice(ppDevice, DriverType, Flags, SDKVersion, FeatureLevels, pFeatureLevels, - ppImmediateContext, NULL, NULL); - } - else if(SUCCEEDED(ret)) - { - RDCLOG("Created wrapped D3D11 device."); - } - else - { - RDCDEBUG("failed. HRESULT: %s", ToStr(ret).c_str()); - } - - return ret; + return d3d11hooks.Create_Internal( + [=](IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, + CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion, + CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, IDXGISwapChain **ppSwapChain, + ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel, + ID3D11DeviceContext **ppImmediateContext) { + // we know that when we come back in here the swapchain parameters will be NULL because + // that's what we pass below + RDCASSERT(!pSwapChainDesc && ppSwapChain); + return d3d11hooks.nvapi_CreateDevice_real( + pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, + ppDevice, pFeatureLevel, ppImmediateContext, outNVLevel); + }, + pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, NULL, + NULL, ppDevice, pFeatureLevel, ppImmediateContext); } static HRESULT WINAPI nvapi_CreateDeviceAndSwapChain( @@ -471,69 +391,18 @@ private: ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel, ID3D11DeviceContext **ppImmediateContext, NVAPI_DEVICE_FEATURE_LEVEL *outNVLevel) { - RDCDEBUG("Call to nvapi_CreateDeviceAndSwapChain"); - - bool reading = RenderDoc::Inst().IsReplayApp(); - - if(reading) - { - RDCDEBUG("In replay app"); - } - - if(!reading && RenderDoc::Inst().GetCaptureOptions().apiValidation) - { - Flags |= D3D11_CREATE_DEVICE_DEBUG; - } - else - { - Flags &= ~D3D11_CREATE_DEVICE_DEBUG; - } - - DXGI_SWAP_CHAIN_DESC swapDesc; - DXGI_SWAP_CHAIN_DESC *pUsedSwapDesc = NULL; - - if(pSwapChainDesc) - { - swapDesc = *pSwapChainDesc; - pUsedSwapDesc = &swapDesc; - } - - if(pUsedSwapDesc && !RenderDoc::Inst().GetCaptureOptions().allowFullscreen) - { - pUsedSwapDesc->Windowed = TRUE; - } - - RDCDEBUG("Calling real createdevice..."); - - HRESULT ret = d3d11hooks.nvapi_CreateDeviceAndSwapChain_real( + return d3d11hooks.Create_Internal( + [=](IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, + CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion, + CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, IDXGISwapChain **ppSwapChain, + ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel, + ID3D11DeviceContext **ppImmediateContext) { + return d3d11hooks.nvapi_CreateDeviceAndSwapChain_real( + pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, + pSwapChainDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext, outNVLevel); + }, pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, - pUsedSwapDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext, outNVLevel); - - RDCDEBUG("Called real createdevice..."); - - bool suppress = false; - - suppress = (Flags & D3D11_CREATE_DEVICE_PREVENT_ALTERING_LAYER_SETTINGS_FROM_REGISTRY) != 0; - - if(suppress && !reading) - { - RDCDEBUG("Application requested not to be hooked."); - } - else if(SUCCEEDED(ret) && ppDevice) - { - d3d11hooks.WrapDevice(ppDevice, DriverType, Flags, SDKVersion, FeatureLevels, pFeatureLevels, - ppImmediateContext, ppSwapChain, pSwapChainDesc); - } - else if(SUCCEEDED(ret)) - { - RDCLOG("Created wrapped D3D11 device."); - } - else - { - RDCDEBUG("failed. HRESULT: %s", ToStr(ret).c_str()); - } - - return ret; + pSwapChainDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext); } static void *nvapi_QueryInterface_hook(uint32_t ID) @@ -613,26 +482,3 @@ private: }; D3D11Hook D3D11Hook::d3d11hooks; - -extern "C" __declspec(dllexport) HRESULT __cdecl RENDERDOC_CreateWrappedD3D11DeviceAndSwapChain( - __in_opt IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, - __in_ecount_opt(FeatureLevels) CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, - UINT SDKVersion, __in_opt CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, - __out_opt IDXGISwapChain **ppSwapChain, __out_opt ID3D11Device **ppDevice, - __out_opt D3D_FEATURE_LEVEL *pFeatureLevel, __out_opt ID3D11DeviceContext **ppImmediateContext) -{ - return D3D11Hook::CreateWrappedDeviceAndSwapChain( - pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, - pSwapChainDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext); -} - -extern "C" __declspec(dllexport) HRESULT __cdecl RENDERDOC_CreateWrappedD3D11Device( - __in_opt IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, - __in_ecount_opt(FeatureLevels) CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, - UINT SDKVersion, __out_opt ID3D11Device **ppDevice, __out_opt D3D_FEATURE_LEVEL *pFeatureLevel, - __out_opt ID3D11DeviceContext **ppImmediateContext) -{ - return D3D11Hook::CreateWrappedDeviceAndSwapChain( - pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, NULL, NULL, - ppDevice, pFeatureLevel, ppImmediateContext); -} diff --git a/renderdoc/driver/d3d11/d3d11_replay.cpp b/renderdoc/driver/d3d11/d3d11_replay.cpp index 007eee43f..02745ad6e 100644 --- a/renderdoc/driver/d3d11/d3d11_replay.cpp +++ b/renderdoc/driver/d3d11/d3d11_replay.cpp @@ -3469,12 +3469,6 @@ void D3D11Replay::SetProxyBufferData(ResourceId bufid, byte *data, size_t dataSi ID3DDevice *GetD3D11DeviceIfAlloc(IUnknown *dev); -extern "C" HRESULT RENDERDOC_CreateWrappedD3D11DeviceAndSwapChain( - __in_opt IDXGIAdapter *, D3D_DRIVER_TYPE, HMODULE, UINT, - __in_ecount_opt(FeatureLevels) CONST D3D_FEATURE_LEVEL *, UINT FeatureLevels, UINT, - __in_opt CONST DXGI_SWAP_CHAIN_DESC *, __out_opt IDXGISwapChain **, __out_opt ID3D11Device **, - __out_opt D3D_FEATURE_LEVEL *, __out_opt ID3D11DeviceContext **); - ReplayStatus D3D11_CreateReplayDevice(RDCFile *rdc, IReplayDriver **driver) { RDCDEBUG("Creating a D3D11 replay device"); @@ -3487,6 +3481,9 @@ ReplayStatus D3D11_CreateReplayDevice(RDCFile *rdc, IReplayDriver **driver) return ReplayStatus::APIInitFailed; } + PFN_D3D11_CREATE_DEVICE_AND_SWAP_CHAIN CreateDeviceAndSwapChain = + (PFN_D3D11_CREATE_DEVICE_AND_SWAP_CHAIN)GetProcAddress(lib, "D3D11CreateDeviceAndSwapChain"); + lib = LoadLibraryA("d3d9.dll"); if(lib == NULL) { @@ -3588,9 +3585,10 @@ ReplayStatus D3D11_CreateReplayDevice(RDCFile *rdc, IReplayDriver **driver) // check for feature level 11 support - passing NULL feature level array implicitly checks for // 11_0 before others - hr = RENDERDOC_CreateWrappedD3D11DeviceAndSwapChain(NULL, D3D_DRIVER_TYPE_HARDWARE, NULL, 0, NULL, - 0, D3D11_SDK_VERSION, NULL, NULL, NULL, - &maxFeatureLevel, NULL); + ID3D11Device *dev = NULL; + hr = CreateDeviceAndSwapChain(NULL, D3D_DRIVER_TYPE_HARDWARE, NULL, 0, NULL, 0, D3D11_SDK_VERSION, + NULL, NULL, &dev, &maxFeatureLevel, NULL); + SAFE_RELEASE(dev); bool warpFallback = false; @@ -3606,7 +3604,15 @@ ReplayStatus D3D11_CreateReplayDevice(RDCFile *rdc, IReplayDriver **driver) hr = E_FAIL; for(;;) { - hr = RENDERDOC_CreateWrappedD3D11DeviceAndSwapChain( +#if ENABLED(RDOC_DEVEL) + // in development builds, always enable debug layer during replay + flags |= D3D11_CREATE_DEVICE_DEBUG; +#else + // in release builds, never enable it + flags &= ~D3D11_CREATE_DEVICE_DEBUG; +#endif + + hr = CreateDeviceAndSwapChain( /*pAdapter=*/NULL, driverType, /*Software=*/NULL, flags, /*pFeatureLevels=*/featureLevelArray, /*nFeatureLevels=*/numFeatureLevels, D3D11_SDK_VERSION, /*pSwapChainDesc=*/NULL, (IDXGISwapChain **)NULL, (ID3D11Device **)&device, @@ -3614,7 +3620,7 @@ ReplayStatus D3D11_CreateReplayDevice(RDCFile *rdc, IReplayDriver **driver) if(SUCCEEDED(hr)) { - WrappedID3D11Device *wrappedDev = (WrappedID3D11Device *)device; + WrappedID3D11Device *wrappedDev = new WrappedID3D11Device(device, initParams); wrappedDev->SetInitParams(initParams, ver); RDCLOG("Created device."); @@ -3671,7 +3677,7 @@ static DriverRegistration D3D11DriverRegistration(RDCDriver::D3D11, &D3D11_Creat void D3D11_ProcessStructured(RDCFile *rdc, SDFile &output) { - WrappedID3D11Device device(NULL, NULL); + WrappedID3D11Device device(NULL, D3D11InitParams()); int sectionIdx = rdc->SectionIndex(SectionType::FrameCapture); diff --git a/renderdoc/driver/d3d12/d3d12_common.cpp b/renderdoc/driver/d3d12/d3d12_common.cpp index bb7e128dc..9504daec8 100644 --- a/renderdoc/driver/d3d12/d3d12_common.cpp +++ b/renderdoc/driver/d3d12/d3d12_common.cpp @@ -98,6 +98,56 @@ void D3D12MarkerRegion::End(ID3D12CommandQueue *queue) queue->EndEvent(); } +bool EnableD3D12DebugLayer(PFN_D3D12_GET_DEBUG_INTERFACE getDebugInterface) +{ + if(!getDebugInterface) + getDebugInterface = (PFN_D3D12_GET_DEBUG_INTERFACE)GetProcAddress(GetModuleHandleA("d3d12.dll"), + "D3D12GetDebugInterface"); + + if(!getDebugInterface) + { + RDCERR("Couldn't find D3D12GetDebugInterface!"); + return false; + } + + ID3D12Debug *debug = NULL; + HRESULT hr = getDebugInterface(__uuidof(ID3D12Debug), (void **)&debug); + + if(SUCCEEDED(hr) && debug) + { + debug->EnableDebugLayer(); + + RDCDEBUG("Enabling debug layer"); + +// enable this to get GPU-based validation, where available, whenever we enable API validation +#if 0 + ID3D12Debug1 *debug1 = NULL; + hr = debug->QueryInterface(__uuidof(ID3D12Debug1), (void **)&debug1); + + if(SUCCEEDED(hr) && debug1) + { + RDCDEBUG("Enabling GPU-based validation"); + debug1->SetEnableGPUBasedValidation(true); + SAFE_RELEASE(debug1); + } + else + { + RDCDEBUG("GPU-based validation not available"); + } +#endif + + SAFE_RELEASE(debug); + + return true; + } + else + { + RDCERR("Couldn't enable debug layer: %x", hr); + } + + return false; +} + D3D12InitParams::D3D12InitParams() { MinimumFeatureLevel = D3D_FEATURE_LEVEL_11_0; diff --git a/renderdoc/driver/d3d12/d3d12_common.h b/renderdoc/driver/d3d12/d3d12_common.h index eaf1720fb..c44e15a18 100644 --- a/renderdoc/driver/d3d12/d3d12_common.h +++ b/renderdoc/driver/d3d12/d3d12_common.h @@ -52,6 +52,8 @@ struct D3D12MarkerRegion ID3D12CommandQueue *queue = NULL; }; +bool EnableD3D12DebugLayer(PFN_D3D12_GET_DEBUG_INTERFACE getDebugInterface = NULL); + inline void SetObjName(ID3D12Object *obj, const std::string &utf8name) { obj->SetName(StringFormat::UTF82Wide(utf8name).c_str()); diff --git a/renderdoc/driver/d3d12/d3d12_device.cpp b/renderdoc/driver/d3d12/d3d12_device.cpp index 2715434fd..2527db5a6 100644 --- a/renderdoc/driver/d3d12/d3d12_device.cpp +++ b/renderdoc/driver/d3d12/d3d12_device.cpp @@ -133,7 +133,7 @@ HRESULT STDMETHODCALLTYPE WrappedID3D12DebugDevice::QueryInterface(REFIID riid, return m_pDebug->QueryInterface(riid, ppvObject); } -WrappedID3D12Device::WrappedID3D12Device(ID3D12Device *realDevice, D3D12InitParams *params, +WrappedID3D12Device::WrappedID3D12Device(ID3D12Device *realDevice, D3D12InitParams params, bool enabledDebugLayer) : m_RefCounter(realDevice, false), m_SoftRefCounter(NULL, false), @@ -307,8 +307,7 @@ WrappedID3D12Device::WrappedID3D12Device(ID3D12Device *realDevice, D3D12InitPara RDCDEBUG("Couldn't get ID3D12InfoQueue."); } - if(params) - m_InitParams = *params; + m_InitParams = params; } WrappedID3D12Device::~WrappedID3D12Device() diff --git a/renderdoc/driver/d3d12/d3d12_device.h b/renderdoc/driver/d3d12/d3d12_device.h index 95c0cd408..9419a8705 100644 --- a/renderdoc/driver/d3d12/d3d12_device.h +++ b/renderdoc/driver/d3d12/d3d12_device.h @@ -374,7 +374,7 @@ public: static const int AllocPoolCount = 4; ALLOCATE_WITH_WRAPPED_POOL(WrappedID3D12Device, AllocPoolCount); - WrappedID3D12Device(ID3D12Device *realDevice, D3D12InitParams *params, bool enabledDebugLayer); + WrappedID3D12Device(ID3D12Device *realDevice, D3D12InitParams params, bool enabledDebugLayer); bool IsDebugLayerEnabled() const { return m_debugLayerEnabled; } virtual ~WrappedID3D12Device(); diff --git a/renderdoc/driver/d3d12/d3d12_hooks.cpp b/renderdoc/driver/d3d12/d3d12_hooks.cpp index 9c2c29849..651bed953 100644 --- a/renderdoc/driver/d3d12/d3d12_hooks.cpp +++ b/renderdoc/driver/d3d12/d3d12_hooks.cpp @@ -123,12 +123,6 @@ public: } void OptionsUpdated(const char *libName) {} - static HRESULT CreateWrappedDevice(IUnknown *pAdapter, D3D_FEATURE_LEVEL MinimumFeatureLevel, - REFIID riid, void **ppDevice) - { - return d3d12hooks.Create_Internal(pAdapter, MinimumFeatureLevel, riid, ppDevice); - } - private: static D3D12Hook d3d12hooks; @@ -184,76 +178,21 @@ private: RDCDEBUG("Call to Create_Internal Feature Level %x", MinimumFeatureLevel, ToStr(riid).c_str()); - bool reading = RenderDoc::Inst().IsReplayApp(); + // we should no longer go through here in the replay application + RDCASSERT(!RenderDoc::Inst().IsReplayApp()); - if(reading) - { - RDCDEBUG("In replay app"); - } + bool EnableDebugLayer = false; - const bool EnableDebugLayer = -// toggle on/off if you want debug layer during replay -#if ENABLED(RDOC_DEVEL) - RenderDoc::Inst().IsReplayApp() || -#endif - (!reading && RenderDoc::Inst().GetCaptureOptions().apiValidation); - - if(EnableDebugLayer) - { - PFN_D3D12_GET_DEBUG_INTERFACE getfn = GetDebugInterface(); - - if(getfn == NULL) - getfn = (PFN_D3D12_GET_DEBUG_INTERFACE)GetProcAddress(GetModuleHandleA("d3d12.dll"), - "D3D12GetDebugInterface"); - - if(getfn) - { - ID3D12Debug *debug = NULL; - HRESULT hr = getfn(__uuidof(ID3D12Debug), (void **)&debug); - - if(SUCCEEDED(hr) && debug) - { - debug->EnableDebugLayer(); - - RDCDEBUG("Enabling debug layer"); - -// enable this to get GPU-based validation, where available, whenever we enable API validation -#if 0 - ID3D12Debug1 *debug1 = NULL; - hr = debug->QueryInterface(__uuidof(ID3D12Debug1), (void **)&debug1); - - if(SUCCEEDED(hr) && debug1) - { - RDCDEBUG("Enabling GPU-based validation"); - debug1->SetEnableGPUBasedValidation(true); - SAFE_RELEASE(debug1); - } - else - { - RDCDEBUG("GPU-based validation not available"); - } -#endif - } - else - { - RDCERR("Couldn't enable debug layer: %x", hr); - } - - SAFE_RELEASE(debug); - } - else - { - RDCERR("Couldn't find D3D12GetDebugInterface!"); - } - } + if(RenderDoc::Inst().GetCaptureOptions().apiValidation) + EnableDebugLayer = EnableD3D12DebugLayer(GetDebugInterface()); RDCDEBUG("Calling real createdevice..."); - PFN_D3D12_CREATE_DEVICE createFunc = - (PFN_D3D12_CREATE_DEVICE)GetProcAddress(GetModuleHandleA("d3d12.dll"), "D3D12CreateDevice"); + PFN_D3D12_CREATE_DEVICE createFunc = CreateDevice(); if(createFunc == NULL) - createFunc = CreateDevice(); + createFunc = (PFN_D3D12_CREATE_DEVICE)GetProcAddress(GetModuleHandleA("d3d12.dll"), + "D3D12CreateDevice"); // shouldn't ever get here, we should either have it from procaddress or the trampoline, but // let's be safe. @@ -297,7 +236,7 @@ private: dev = (ID3D12Device *)dev1; } - WrappedID3D12Device *wrap = new WrappedID3D12Device(dev, ¶ms, EnableDebugLayer); + WrappedID3D12Device *wrap = new WrappedID3D12Device(dev, params, EnableDebugLayer); RDCDEBUG("created wrapped device."); @@ -365,11 +304,3 @@ private: }; D3D12Hook D3D12Hook::d3d12hooks; - -extern "C" __declspec(dllexport) HRESULT - __cdecl RENDERDOC_CreateWrappedD3D12Device(IUnknown *pAdapter, - D3D_FEATURE_LEVEL MinimumFeatureLevel, REFIID riid, - void **ppDevice) -{ - return D3D12Hook::CreateWrappedDevice(pAdapter, MinimumFeatureLevel, riid, ppDevice); -} \ No newline at end of file diff --git a/renderdoc/driver/d3d12/d3d12_replay.cpp b/renderdoc/driver/d3d12/d3d12_replay.cpp index 836cf96b8..c40332c42 100644 --- a/renderdoc/driver/d3d12/d3d12_replay.cpp +++ b/renderdoc/driver/d3d12/d3d12_replay.cpp @@ -40,12 +40,6 @@ static const char *LiveDriverDisassemblyTarget = "Live driver disassembly"; -extern "C" __declspec(dllexport) HRESULT - __cdecl RENDERDOC_CreateWrappedDXGIFactory1(REFIID riid, void **ppFactory); -extern "C" __declspec(dllexport) HRESULT - __cdecl RENDERDOC_CreateWrappedD3D12Device(IUnknown *pAdapter, - D3D_FEATURE_LEVEL MinimumFeatureLevel, REFIID riid, - void **ppDevice); ID3DDevice *GetD3D12DeviceIfAlloc(IUnknown *dev); static const char *DXBCDisassemblyTarget = "DXBC"; @@ -82,7 +76,12 @@ void D3D12Replay::CreateResources() { if(RenderDoc::Inst().IsReplayApp()) { - HRESULT hr = RENDERDOC_CreateWrappedDXGIFactory1(__uuidof(IDXGIFactory4), (void **)&m_pFactory); + typedef HRESULT(WINAPI * PFN_CREATE_DXGI_FACTORY)(REFIID, void **); + + PFN_CREATE_DXGI_FACTORY createFunc = + (PFN_CREATE_DXGI_FACTORY)GetProcAddress(GetModuleHandleA("dxgi.dll"), "CreateDXGIFactory1"); + + HRESULT hr = createFunc(__uuidof(IDXGIFactory4), (void **)&m_pFactory); if(FAILED(hr)) { @@ -91,6 +90,8 @@ void D3D12Replay::CreateResources() if(m_pFactory) { + RefCountDXGIObject::HandleWrap(__uuidof(IDXGIFactory4), (void **)&m_pFactory); + LUID luid = m_pDevice->GetAdapterLuid(); IDXGIAdapter *pDXGIAdapter; @@ -3444,6 +3445,9 @@ ReplayStatus D3D12_CreateReplayDevice(RDCFile *rdc, IReplayDriver **driver) return ReplayStatus::APIInitFailed; } + PFN_D3D12_CREATE_DEVICE createDevice = + (PFN_D3D12_CREATE_DEVICE)GetProcAddress(lib, "D3D12CreateDevice"); + lib = LoadLibraryA("dxgi.dll"); if(lib == NULL) { @@ -3507,9 +3511,16 @@ ReplayStatus D3D12_CreateReplayDevice(RDCFile *rdc, IReplayDriver **driver) if(!rgp->Initialised()) SAFE_DELETE(rgp); + bool EnableDebugLayer = false; + +#if ENABLED(RDOC_DEVEL) + // in development builds, always enable debug layer during replay + EnableDebugLayer = EnableD3D12DebugLayer(); +#endif + ID3D12Device *dev = NULL; - HRESULT hr = RENDERDOC_CreateWrappedD3D12Device(NULL, initParams.MinimumFeatureLevel, - __uuidof(ID3D12Device), (void **)&dev); + HRESULT hr = + createDevice(NULL, initParams.MinimumFeatureLevel, __uuidof(ID3D12Device), (void **)&dev); if(FAILED(hr)) { @@ -3520,7 +3531,7 @@ ReplayStatus D3D12_CreateReplayDevice(RDCFile *rdc, IReplayDriver **driver) return ReplayStatus::APIHardwareUnsupported; } - WrappedID3D12Device *wrappedDev = (WrappedID3D12Device *)dev; + WrappedID3D12Device *wrappedDev = new WrappedID3D12Device(dev, initParams, EnableDebugLayer); wrappedDev->SetInitParams(initParams, ver); RDCLOG("Created device."); @@ -3537,7 +3548,7 @@ static DriverRegistration D3D12DriverRegistration(RDCDriver::D3D12, &D3D12_Creat void D3D12_ProcessStructured(RDCFile *rdc, SDFile &output) { - WrappedID3D12Device device(NULL, NULL, false); + WrappedID3D12Device device(NULL, D3D12InitParams(), false); int sectionIdx = rdc->SectionIndex(SectionType::FrameCapture); diff --git a/renderdoc/driver/dxgi/dxgi_hooks.cpp b/renderdoc/driver/dxgi/dxgi_hooks.cpp index 3d8832fee..ff0df3fd4 100644 --- a/renderdoc/driver/dxgi/dxgi_hooks.cpp +++ b/renderdoc/driver/dxgi/dxgi_hooks.cpp @@ -74,12 +74,7 @@ struct RenderDocAnalysis : IDXGraphicsAnalysis class DXGIHook : LibraryHook { public: - DXGIHook() - { - LibraryHooks::GetInstance().RegisterHook(DLL_NAME, this); - m_HasHooks = false; - } - + DXGIHook() { LibraryHooks::GetInstance().RegisterHook(DLL_NAME, this); } bool CreateHooks(const char *libName) { bool success = true; @@ -94,81 +89,13 @@ public: if(!success) return false; - m_HasHooks = true; - return true; } void OptionsUpdated(const char *libName) {} - static HRESULT CreateWrappedFactory1(REFIID riid, void **ppFactory) - { - if(dxgihooks.m_HasHooks) - return dxgihooks.CreateDXGIFactory1_hook(riid, ppFactory); - - HMODULE dxgi = GetModuleHandleA("dxgi.dll"); - - if(dxgi) - { - PFN_CREATE_DXGI_FACTORY createFunc = - (PFN_CREATE_DXGI_FACTORY)GetProcAddress(dxgi, "CreateDXGIFactory1"); - - if(!createFunc) - { - RDCERR("Trying to create hooked dxgi factory without dxgi loaded"); - return E_INVALIDARG; - } - - HRESULT ret = createFunc(riid, ppFactory); - - if(SUCCEEDED(ret)) - RefCountDXGIObject::HandleWrap(riid, ppFactory); - - return ret; - } - else - { - RDCERR("Something went seriously wrong, dxgi.dll couldn't be loaded!"); - return E_UNEXPECTED; - } - } - - static HRESULT CreateWrappedFactory2(UINT Flags, REFIID riid, void **ppFactory) - { - if(dxgihooks.m_HasHooks) - return dxgihooks.CreateDXGIFactory2_hook(Flags, riid, ppFactory); - - HMODULE dxgi = GetModuleHandleA("dxgi.dll"); - - if(dxgi) - { - PFN_CREATE_DXGI_FACTORY2 createFunc = - (PFN_CREATE_DXGI_FACTORY2)GetProcAddress(dxgi, "CreateDXGIFactory2"); - - if(!createFunc) - { - RDCERR("Trying to create hooked dxgi factory without dxgi loaded"); - return E_INVALIDARG; - } - - HRESULT ret = createFunc(Flags, riid, ppFactory); - - if(SUCCEEDED(ret)) - RefCountDXGIObject::HandleWrap(riid, ppFactory); - - return ret; - } - else - { - RDCERR("Something went seriously wrong, dxgi.dll couldn't be loaded!"); - return E_UNEXPECTED; - } - } - private: static DXGIHook dxgihooks; - bool m_HasHooks; - RenderDocAnalysis m_RenderDocAnalysis; Hook CreateDXGIFactory; @@ -251,15 +178,3 @@ private: }; DXGIHook DXGIHook::dxgihooks; - -extern "C" __declspec(dllexport) HRESULT - __cdecl RENDERDOC_CreateWrappedDXGIFactory1(REFIID riid, void **ppFactory) -{ - return DXGIHook::CreateWrappedFactory1(riid, ppFactory); -} - -extern "C" __declspec(dllexport) HRESULT - __cdecl RENDERDOC_CreateWrappedDXGIFactory2(UINT Flags, REFIID riid, void **ppFactory) -{ - return DXGIHook::CreateWrappedFactory2(Flags, riid, ppFactory); -}