diff --git a/renderdoc/driver/d3d11/d3d11_hooks.cpp b/renderdoc/driver/d3d11/d3d11_hooks.cpp index 18d885795..addb9f006 100644 --- a/renderdoc/driver/d3d11/d3d11_hooks.cpp +++ b/renderdoc/driver/d3d11/d3d11_hooks.cpp @@ -23,6 +23,7 @@ * THE SOFTWARE. ******************************************************************************/ +#include "d3d11_hooks.h" #include "driver/dxgi/dxgi_wrapped.h" #include "driver/ihv/amd/official/DXExt/AmdDxExtApi.h" #include "hooks/hooks.h" @@ -38,8 +39,6 @@ #endif -ID3D11Resource *UnwrapDXResource(void *dxObject); - ID3DDevice *GetD3D11DeviceIfAlloc(IUnknown *dev) { if(WrappedID3D11Device::IsAlloc(dev)) @@ -48,78 +47,9 @@ ID3DDevice *GetD3D11DeviceIfAlloc(IUnknown *dev) return NULL; } -enum NVAPI_DEVICE_FEATURE_LEVEL -{ - dummy = 0, -}; - -typedef void *(__cdecl *PFNNVQueryInterface)(uint32_t ID); -typedef HRESULT (*PFNNVCreateDevice)(_In_opt_ IDXGIAdapter *, D3D_DRIVER_TYPE, HMODULE, UINT, - _In_reads_opt_(FeatureLevels) CONST D3D_FEATURE_LEVEL *, - UINT FeatureLevels, UINT, _COM_Outptr_opt_ ID3D11Device **, - _Out_opt_ D3D_FEATURE_LEVEL *, - _COM_Outptr_opt_ ID3D11DeviceContext **, - NVAPI_DEVICE_FEATURE_LEVEL *); -typedef HRESULT (*PFNNVCreateDeviceAndSwapChain)( - _In_opt_ IDXGIAdapter *, D3D_DRIVER_TYPE, HMODULE, UINT, - _In_reads_opt_(FeatureLevels) CONST D3D_FEATURE_LEVEL *, UINT FeatureLevels, UINT, - _In_opt_ CONST DXGI_SWAP_CHAIN_DESC *, _COM_Outptr_opt_ IDXGISwapChain **, - _COM_Outptr_opt_ ID3D11Device **, _Out_opt_ D3D_FEATURE_LEVEL *, - _COM_Outptr_opt_ ID3D11DeviceContext **, NVAPI_DEVICE_FEATURE_LEVEL *); - -// this is the type of the lambda we use to route the call out to the 'real' function inside our -// generic wrapper. -// Could be any of D3D11CreateDevice, D3D11CreateDeviceAndSwapChain, or the nvapi equivalents -typedef std::function - RealFunction; - -#define NVENCAPI __stdcall -enum NVENCSTATUS -{ - NV_ENC_SUCCESS = 0, - NV_ENC_ERR_INVALID_PTR = 6, -}; - -enum NV_ENC_INPUT_RESOURCE_TYPE -{ - NV_ENC_INPUT_RESOURCE_TYPE_DIRECTX = 0x0, - NV_ENC_INPUT_RESOURCE_TYPE_CUDADEVICEPTR = 0x1, - NV_ENC_INPUT_RESOURCE_TYPE_CUDAARRAY = 0x2, - NV_ENC_INPUT_RESOURCE_TYPE_OPENGL_TEX = 0x3 -}; - -struct NV_ENC_REGISTER_RESOURCE -{ - uint32_t version; - NV_ENC_INPUT_RESOURCE_TYPE resourceType; - uint32_t dummy[4]; - void *resourceToRegister; - - // there is more data here but we don't allocate this structure only patch the above pointer, so - // we don't need it -}; - -typedef NVENCSTATUS(NVENCAPI *PNVENCREGISTERRESOURCE)(void *, NV_ENC_REGISTER_RESOURCE *params); - -struct NV_ENCODE_API_FUNCTION_LIST -{ - uint32_t version; - uint32_t reserved; - void *otherFunctions[30]; // other functions in the dispatch table - PNVENCREGISTERRESOURCE nvEncRegisterResource; - - // there is more data here but we don't allocate this structure only patch the above pointer, so - // we don't need it -}; - -typedef NVENCSTATUS(NVENCAPI *PFN_NvEncodeAPICreateInstance)(NV_ENCODE_API_FUNCTION_LIST *); - class D3D11Hook : LibraryHook { public: - D3D11Hook() { m_InsideCreate = false; } void RegisterHooks() { RDCLOG("Registering D3D11 hooks"); @@ -144,18 +74,6 @@ public: LibraryHooks::RegisterLibraryHook(BIT_SPECIFIC_DLL("atidxx32.dll", "atidxx64.dll"), NULL); AmdCreate11.Register(BIT_SPECIFIC_DLL("atidxx32.dll", "atidxx64.dll"), "AmdDxExtCreate11", AmdCreate11_hook); - - // nvapi has some seriously awful ""feature"" where it can wrap CreateDevice with some other - // extra parameter. Since we don't want to hook nvapi when it calls out to d3d11.dll, we have to - // intercept it here. - LibraryHooks::RegisterLibraryHook(BIT_SPECIFIC_DLL("nvapi.dll", "nvapi64.dll"), NULL); - nvapi_QueryInterface.Register(BIT_SPECIFIC_DLL("nvapi.dll", "nvapi64.dll"), - "nvapi_QueryInterface", nvapi_QueryInterface_hook); - - // we need to wrap nvcodec to handle unwrapping D3D11 pointers passed to it - LibraryHooks::RegisterLibraryHook(BIT_SPECIFIC_DLL("nvEncodeAPI.dll", "nvEncodeAPI64.dll"), NULL); - NvEncodeCreate.Register(BIT_SPECIFIC_DLL("nvEncodeAPI.dll", "nvEncodeAPI64.dll"), - "NvEncodeAPICreateInstance", NvEncodeAPICreateInstance_hook); } private: @@ -166,13 +84,22 @@ private: // optional extension hooks HookedFunction AmdCreate11; - HookedFunction nvapi_QueryInterface; - HookedFunction NvEncodeCreate; // re-entrancy detection (can happen in rare cases with e.g. fraps) - bool m_InsideCreate; + bool m_InsideCreate = false; - HRESULT Create_Internal(RealFunction real, __in_opt IDXGIAdapter *pAdapter, + friend HRESULT CreateD3D11_Internal(RealD3D11CreateFunction real, __in_opt IDXGIAdapter *pAdapter, + D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, + __in_ecount_opt(FeatureLevels) + CONST D3D_FEATURE_LEVEL *pFeatureLevels, + UINT FeatureLevels, UINT SDKVersion, + __in_opt CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, + __out_opt IDXGISwapChain **ppSwapChain, + __out_opt ID3D11Device **ppDevice, + __out_opt D3D_FEATURE_LEVEL *pFeatureLevel, + __out_opt ID3D11DeviceContext **ppImmediateContext); + + HRESULT Create_Internal(RealD3D11CreateFunction real, __in_opt IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, __in_ecount_opt(FeatureLevels) CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion, @@ -335,153 +262,21 @@ private: return E_FAIL; } - - PFNNVCreateDevice nvapi_CreateDevice_real = NULL; - PFNNVCreateDeviceAndSwapChain nvapi_CreateDeviceAndSwapChain_real = NULL; - - static HRESULT WINAPI nvapi_CreateDevice(IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, - HMODULE Software, UINT Flags, - CONST D3D_FEATURE_LEVEL *pFeatureLevels, - UINT FeatureLevels, UINT SDKVersion, - ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel, - ID3D11DeviceContext **ppImmediateContext, - NVAPI_DEVICE_FEATURE_LEVEL *outNVLevel) - { - return d3d11hooks.Create_Internal( - [=](IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, - CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion, - CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, IDXGISwapChain **ppSwapChain, - ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel, - ID3D11DeviceContext **ppImmediateContext) { - // we know that when we come back in here the swapchain parameters will be NULL because - // that's what we pass below - RDCASSERT(!pSwapChainDesc && !ppSwapChain); - return d3d11hooks.nvapi_CreateDevice_real( - pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, - ppDevice, pFeatureLevel, ppImmediateContext, outNVLevel); - }, - pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, NULL, - NULL, ppDevice, pFeatureLevel, ppImmediateContext); - } - - static HRESULT WINAPI nvapi_CreateDeviceAndSwapChain( - IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, - CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion, - CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, IDXGISwapChain **ppSwapChain, - ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel, - ID3D11DeviceContext **ppImmediateContext, NVAPI_DEVICE_FEATURE_LEVEL *outNVLevel) - { - return d3d11hooks.Create_Internal( - [=](IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, - CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion, - CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, IDXGISwapChain **ppSwapChain, - ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel, - ID3D11DeviceContext **ppImmediateContext) { - return d3d11hooks.nvapi_CreateDeviceAndSwapChain_real( - pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, - pSwapChainDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext, outNVLevel); - }, - pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, - pSwapChainDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext); - } - - static void *nvapi_QueryInterface_hook(uint32_t ID) - { - void *real = d3d11hooks.nvapi_QueryInterface()(ID); - - if(real == NULL) - return real; - - if(ID == 0x6A16D3A0) - { - d3d11hooks.nvapi_CreateDevice_real = (PFNNVCreateDevice)real; - return &nvapi_CreateDevice; - } - else if(ID == 0xBB939EE5) - { - d3d11hooks.nvapi_CreateDeviceAndSwapChain_real = (PFNNVCreateDeviceAndSwapChain)real; - return &nvapi_CreateDeviceAndSwapChain; - } - else if(ID == 0xAD298D3F || ID == 0x0150E828 || ID == 0x33C7358C || ID == 0x593E8644) - { - // these seem to be fetched inside NvAPI_Initialize so we allow them through to avoid - // causing problems - return real; - } - else - { - if(RenderDoc::Inst().IsVendorExtensionEnabled(VendorExtensions::NvAPI)) - { - RDCDEBUG("NvAPI allowed: Returning %p for nvapi_QueryInterface(%x)", real, ID); - return real; - } - else - { - static int count = 0; - if(count < 10) - RDCWARN("NvAPI disabled: Returning NULL for nvapi_QueryInterface(%x)", ID); - count++; - return NULL; - } - } - } - - PNVENCREGISTERRESOURCE real_nvEncRegisterResource = NULL; - - static NVENCSTATUS NVENCAPI NvEncodeAPIRegisterResource_hook(void *encoder, - NV_ENC_REGISTER_RESOURCE *params) - { - if(!d3d11hooks.real_nvEncRegisterResource) - { - RDCERR("nvEncRegisterResource called without hooking NvEncodeAPICreateInstance!"); - return NV_ENC_ERR_INVALID_PTR; - } - - // only directx textures need to be unwrapped - if(!encoder || !params || params->resourceType != NV_ENC_INPUT_RESOURCE_TYPE_DIRECTX) - return d3d11hooks.real_nvEncRegisterResource(encoder, params); - - // attempt to unwrap the handle in place - void *origHandle = params->resourceToRegister; - params->resourceToRegister = UnwrapDXResource(origHandle); - if(params->resourceToRegister == NULL) - { - RDCERR("Failed to unwrap DX handle %p, falling back to pass-through", origHandle); - params->resourceToRegister = origHandle; - } - - // call out to the actual function - NVENCSTATUS ret = d3d11hooks.real_nvEncRegisterResource(encoder, params); - - // restore the handle to the original value - params->resourceToRegister = origHandle; - - return ret; - } - - static NVENCSTATUS NVENCAPI NvEncodeAPICreateInstance_hook(NV_ENCODE_API_FUNCTION_LIST *functions) - { - NVENCSTATUS ret = d3d11hooks.NvEncodeCreate()(functions); - - if(ret == NV_ENC_SUCCESS && functions && functions->nvEncRegisterResource) - { - // this is an encoded struct version, 7 is a magic value, 8.1 is the major.minor of nvcodec - // and 2 is the struct version - const uint32_t expectedVersion = 7 << 28 | 8 << 1 | 1 << 24 | 2 << 16; - if(functions->version != expectedVersion) - RDCWARN("Call to NvEncodeAPICreateInstance with version %x, expected %x", - functions->version, expectedVersion); - - // we don't handle multiple different pointers coming back, but that seems unlikely. - RDCASSERT(d3d11hooks.real_nvEncRegisterResource == NULL || - d3d11hooks.real_nvEncRegisterResource == functions->nvEncRegisterResource); - d3d11hooks.real_nvEncRegisterResource = functions->nvEncRegisterResource; - - functions->nvEncRegisterResource = &NvEncodeAPIRegisterResource_hook; - } - - return ret; - } }; D3D11Hook D3D11Hook::d3d11hooks; + +HRESULT CreateD3D11_Internal(RealD3D11CreateFunction real, __in_opt IDXGIAdapter *pAdapter, + D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, + __in_ecount_opt(FeatureLevels) CONST D3D_FEATURE_LEVEL *pFeatureLevels, + UINT FeatureLevels, UINT SDKVersion, + __in_opt CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, + __out_opt IDXGISwapChain **ppSwapChain, + __out_opt ID3D11Device **ppDevice, + __out_opt D3D_FEATURE_LEVEL *pFeatureLevel, + __out_opt ID3D11DeviceContext **ppImmediateContext) +{ + return D3D11Hook::d3d11hooks.Create_Internal( + real, pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, + pSwapChainDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext); +} diff --git a/renderdoc/driver/d3d11/d3d11_hooks.h b/renderdoc/driver/d3d11/d3d11_hooks.h new file mode 100644 index 000000000..f03d014ea --- /dev/null +++ b/renderdoc/driver/d3d11/d3d11_hooks.h @@ -0,0 +1,47 @@ +/****************************************************************************** + * The MIT License (MIT) + * + * Copyright (c) 2020 Baldur Karlsson + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + ******************************************************************************/ + +#pragma once + +#include +#include "driver/dx/official/d3d11.h" +#include "driver/dx/official/dxgi.h" + +// this is the type of the lambda we use to route the call out to the 'real' function inside our +// generic wrapper. +// Could be any of D3D11CreateDevice, D3D11CreateDeviceAndSwapChain, or the nvapi equivalents +typedef std::function + RealD3D11CreateFunction; + +HRESULT CreateD3D11_Internal(RealD3D11CreateFunction real, __in_opt IDXGIAdapter *pAdapter, + D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, + __in_ecount_opt(FeatureLevels) CONST D3D_FEATURE_LEVEL *pFeatureLevels, + UINT FeatureLevels, UINT SDKVersion, + __in_opt CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, + __out_opt IDXGISwapChain **ppSwapChain, + __out_opt ID3D11Device **ppDevice, + __out_opt D3D_FEATURE_LEVEL *pFeatureLevel, + __out_opt ID3D11DeviceContext **ppImmediateContext); diff --git a/renderdoc/driver/d3d11/renderdoc_d3d11.vcxproj b/renderdoc/driver/d3d11/renderdoc_d3d11.vcxproj index 18e379fbe..992a34dda 100644 --- a/renderdoc/driver/d3d11/renderdoc_d3d11.vcxproj +++ b/renderdoc/driver/d3d11/renderdoc_d3d11.vcxproj @@ -142,6 +142,7 @@ + diff --git a/renderdoc/driver/d3d11/renderdoc_d3d11.vcxproj.filters b/renderdoc/driver/d3d11/renderdoc_d3d11.vcxproj.filters index 6e943d977..0b0c771e3 100644 --- a/renderdoc/driver/d3d11/renderdoc_d3d11.vcxproj.filters +++ b/renderdoc/driver/d3d11/renderdoc_d3d11.vcxproj.filters @@ -185,5 +185,6 @@ Wrapped + \ No newline at end of file diff --git a/renderdoc/driver/ihv/nv/NV.vcxproj b/renderdoc/driver/ihv/nv/NV.vcxproj index ba19262d6..388b15be4 100644 --- a/renderdoc/driver/ihv/nv/NV.vcxproj +++ b/renderdoc/driver/ihv/nv/NV.vcxproj @@ -96,10 +96,13 @@ + + + diff --git a/renderdoc/driver/ihv/nv/NV.vcxproj.filters b/renderdoc/driver/ihv/nv/NV.vcxproj.filters index 8a52f4619..5737ef821 100644 --- a/renderdoc/driver/ihv/nv/NV.vcxproj.filters +++ b/renderdoc/driver/ihv/nv/NV.vcxproj.filters @@ -7,6 +7,12 @@ Counters + + official\nvapi + + + official\nvapi + @@ -21,10 +27,19 @@ {630ba7bc-f1cd-4dd7-ba0d-5e4d689be9fd} + + {fb35dcda-304a-43af-abfd-86e81fb701ae} + + + {f9b6bd25-7b71-4814-b8de-ca40545419a9} + Counters + + nvapi + \ No newline at end of file diff --git a/renderdoc/driver/ihv/nv/nvapi_hooks.cpp b/renderdoc/driver/ihv/nv/nvapi_hooks.cpp new file mode 100644 index 000000000..9efef9225 --- /dev/null +++ b/renderdoc/driver/ihv/nv/nvapi_hooks.cpp @@ -0,0 +1,291 @@ +/****************************************************************************** + * The MIT License (MIT) + * + * Copyright (c) 2020 Baldur Karlsson + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + ******************************************************************************/ + +#include "driver/d3d11/d3d11_device.h" +#include "driver/d3d11/d3d11_hooks.h" +#include "hooks/hooks.h" + +#include "official/nvapi/nvapi.h" +#include "official/nvapi/nvapi_interface.h" + +#if ENABLED(RDOC_X64) + +#define BIT_SPECIFIC_DLL(dll32, dll64) dll64 + +#else + +#define BIT_SPECIFIC_DLL(dll32, dll64) dll32 + +#endif + +ID3D11Resource *UnwrapDXResource(void *dxObject); + +typedef void *(__cdecl *PFNNVQueryInterface)(uint32_t ID); + +typedef HRESULT (*PFNNVCreateDeviceAndSwapChain)( + _In_opt_ IDXGIAdapter *, D3D_DRIVER_TYPE, HMODULE, UINT, + _In_reads_opt_(FeatureLevels) CONST D3D_FEATURE_LEVEL *, UINT FeatureLevels, UINT, + _In_opt_ CONST DXGI_SWAP_CHAIN_DESC *, _COM_Outptr_opt_ IDXGISwapChain **, + _COM_Outptr_opt_ ID3D11Device **, _Out_opt_ D3D_FEATURE_LEVEL *, + _COM_Outptr_opt_ ID3D11DeviceContext **, NVAPI_DEVICE_FEATURE_LEVEL *); + +// this is the type of the lambda we use to route the call out to the 'real' function inside our +// generic wrapper. +// Could be any of D3D11CreateDevice, D3D11CreateDeviceAndSwapChain, or the nvapi equivalents +typedef std::function + RealD3D11CreateFunction; + +#define NVENCAPI __stdcall +enum NVENCSTATUS +{ + NV_ENC_SUCCESS = 0, + NV_ENC_ERR_INVALID_PTR = 6, +}; + +enum NV_ENC_INPUT_RESOURCE_TYPE +{ + NV_ENC_INPUT_RESOURCE_TYPE_DIRECTX = 0x0, + NV_ENC_INPUT_RESOURCE_TYPE_CUDADEVICEPTR = 0x1, + NV_ENC_INPUT_RESOURCE_TYPE_CUDAARRAY = 0x2, + NV_ENC_INPUT_RESOURCE_TYPE_OPENGL_TEX = 0x3 +}; + +struct NV_ENC_REGISTER_RESOURCE +{ + uint32_t version; + NV_ENC_INPUT_RESOURCE_TYPE resourceType; + uint32_t dummy[4]; + void *resourceToRegister; + + // there is more data here but we don't allocate this structure only patch the above pointer, so + // we don't need it +}; + +typedef NVENCSTATUS(NVENCAPI *PNVENCREGISTERRESOURCE)(void *, NV_ENC_REGISTER_RESOURCE *params); + +struct NV_ENCODE_API_FUNCTION_LIST +{ + uint32_t version; + uint32_t reserved; + void *otherFunctions[30]; // other functions in the dispatch table + PNVENCREGISTERRESOURCE nvEncRegisterResource; + + // there is more data here but we don't allocate this structure only patch the above pointer, so + // we don't need it +}; + +typedef NVENCSTATUS(NVENCAPI *PFN_NvEncodeAPICreateInstance)(NV_ENCODE_API_FUNCTION_LIST *); + +class NVHook : LibraryHook +{ +public: + void RegisterHooks() + { + RDCLOG("Registering nvidia hooks"); + + LibraryHooks::RegisterLibraryHook(BIT_SPECIFIC_DLL("nvapi.dll", "nvapi64.dll"), NULL); + nvapi_QueryInterface.Register(BIT_SPECIFIC_DLL("nvapi.dll", "nvapi64.dll"), + "nvapi_QueryInterface", nvapi_QueryInterface_hook); + + // we need to wrap nvcodec to handle unwrapping D3D11 pointers passed to it + LibraryHooks::RegisterLibraryHook(BIT_SPECIFIC_DLL("nvEncodeAPI.dll", "nvEncodeAPI64.dll"), NULL); + NvEncodeCreate.Register(BIT_SPECIFIC_DLL("nvEncodeAPI.dll", "nvEncodeAPI64.dll"), + "NvEncodeAPICreateInstance", NvEncodeAPICreateInstance_hook); + + // we need ID -> function lookup, not function -> ID as the table gives us. We also want fairly + // quick lookup since some programs call nvapi_QueryInterface at *high* frequency. + for(const NVAPI_INTERFACE_TABLE &iface : nvapi_interface_table) + nvapi_lookup[iface.id] = iface.func; + } + +private: + static NVHook nvhooks; + + std::unordered_map nvapi_lookup; + + HookedFunction nvapi_QueryInterface; + HookedFunction NvEncodeCreate; + +#define HOOK_NVAPI(fname, ID) HookedFunction fname; +#define WHITELIST_NVAPI(fname, ID) + +#define NVAPI_FUNCS() \ + HOOK_NVAPI(NvAPI_D3D11_CreateDevice, 0x6A16D3A0); \ + HOOK_NVAPI(NvAPI_D3D11_CreateDeviceAndSwapChain, 0xBB939EE5); \ + WHITELIST_NVAPI(NvAPI_Initialize, 0x0150E828); + + NVAPI_FUNCS(); + + static HRESULT WINAPI NvAPI_D3D11_CreateDevice_hook( + IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, + CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion, + ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel, + ID3D11DeviceContext **ppImmediateContext, NVAPI_DEVICE_FEATURE_LEVEL *outNVLevel) + { + return CreateD3D11_Internal( + [=](IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, + CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion, + CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, IDXGISwapChain **ppSwapChain, + ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel, + ID3D11DeviceContext **ppImmediateContext) { + // we know that when we come back in here the swapchain parameters will be NULL because + // that's what we pass below + RDCASSERT(!pSwapChainDesc && !ppSwapChain); + return nvhooks.NvAPI_D3D11_CreateDevice()( + pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, + ppDevice, pFeatureLevel, ppImmediateContext, outNVLevel); + }, + pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, NULL, + NULL, ppDevice, pFeatureLevel, ppImmediateContext); + } + + static HRESULT WINAPI NvAPI_D3D11_CreateDeviceAndSwapChain_hook( + IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, + CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion, + CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, IDXGISwapChain **ppSwapChain, + ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel, + ID3D11DeviceContext **ppImmediateContext, NVAPI_DEVICE_FEATURE_LEVEL *outNVLevel) + { + return CreateD3D11_Internal( + [=](IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags, + CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion, + CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, IDXGISwapChain **ppSwapChain, + ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel, + ID3D11DeviceContext **ppImmediateContext) { + return nvhooks.NvAPI_D3D11_CreateDeviceAndSwapChain()( + pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, + pSwapChainDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext, outNVLevel); + }, + pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, + pSwapChainDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext); + } + + static void *nvapi_QueryInterface_hook(uint32_t ID) + { + void *real = nvhooks.nvapi_QueryInterface()(ID); + + if(real == NULL) + return real; + +#undef HOOK_NVAPI +#define HOOK_NVAPI(fname, ID) \ + case ID: \ + { \ + nvhooks.fname.SetFuncPtr(real); \ + return &CONCAT(fname, _hook); \ + } +#undef WHITELIST_NVAPI +#define WHITELIST_NVAPI(fname, ID) \ + case ID: return real; + + switch(ID) + { + NVAPI_FUNCS(); + // unknown, but these are fetched inside NvAPI_Initialize so allow them through to avoid + // causing problems. + WHITELIST_NVAPI(unknown_func, 0xAD298D3F); + WHITELIST_NVAPI(unknown_func, 0x33C7358C); + WHITELIST_NVAPI(unknown_func, 0x593E8644); + default: break; + } + + const char *cname = nvhooks.nvapi_lookup[ID]; + rdcstr name = cname ? cname : StringFormat::Fmt("0x%x", ID); + + if(RenderDoc::Inst().IsVendorExtensionEnabled(VendorExtensions::NvAPI)) + { + RDCDEBUG("NvAPI allowed: Returning %p for nvapi_QueryInterface(%s)", real, name.c_str()); + return real; + } + else + { + static int count = 0; + if(count < 10) + RDCWARN("NvAPI disabled: Returning NULL for nvapi_QueryInterface(%s)", name.c_str()); + count++; + return NULL; + } + } + + PNVENCREGISTERRESOURCE real_nvEncRegisterResource = NULL; + + static NVENCSTATUS NVENCAPI NvEncodeAPIRegisterResource_hook(void *encoder, + NV_ENC_REGISTER_RESOURCE *params) + { + if(!nvhooks.real_nvEncRegisterResource) + { + RDCERR("nvEncRegisterResource called without hooking NvEncodeAPICreateInstance!"); + return NV_ENC_ERR_INVALID_PTR; + } + + // only directx textures need to be unwrapped + if(!encoder || !params || params->resourceType != NV_ENC_INPUT_RESOURCE_TYPE_DIRECTX) + return nvhooks.real_nvEncRegisterResource(encoder, params); + + // attempt to unwrap the handle in place + void *origHandle = params->resourceToRegister; + params->resourceToRegister = UnwrapDXResource(origHandle); + if(params->resourceToRegister == NULL) + { + RDCERR("Failed to unwrap DX handle %p, falling back to pass-through", origHandle); + params->resourceToRegister = origHandle; + } + + // call out to the actual function + NVENCSTATUS ret = nvhooks.real_nvEncRegisterResource(encoder, params); + + // restore the handle to the original value + params->resourceToRegister = origHandle; + + return ret; + } + + static NVENCSTATUS NVENCAPI NvEncodeAPICreateInstance_hook(NV_ENCODE_API_FUNCTION_LIST *functions) + { + NVENCSTATUS ret = nvhooks.NvEncodeCreate()(functions); + + if(ret == NV_ENC_SUCCESS && functions && functions->nvEncRegisterResource) + { + // this is an encoded struct version, 7 is a magic value, 8.1 is the major.minor of nvcodec + // and 2 is the struct version + const uint32_t expectedVersion = 7 << 28 | 8 << 1 | 1 << 24 | 2 << 16; + if(functions->version != expectedVersion) + RDCWARN("Call to NvEncodeAPICreateInstance with version %x, expected %x", + functions->version, expectedVersion); + + // we don't handle multiple different pointers coming back, but that seems unlikely. + RDCASSERT(nvhooks.real_nvEncRegisterResource == NULL || + nvhooks.real_nvEncRegisterResource == functions->nvEncRegisterResource); + nvhooks.real_nvEncRegisterResource = functions->nvEncRegisterResource; + + functions->nvEncRegisterResource = &NvEncodeAPIRegisterResource_hook; + } + + return ret; + } +}; + +NVHook NVHook::nvhooks;