lee/renderdoc
1
0
Fork 0
mirror of https://github.com/baldurk/renderdoc.git synced 2026-05-29 13:20:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move existing minimal nvapi/nvencode hooks to NV project

Browse Source
This commit is contained in:
baldurk
2020-10-06 11:48:09 +01:00
parent f9deead680
commit 351e61b849
7 changed files with 387 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files
renderdoc/driver/d3d11/d3d11_hooks.cpp
+29 -234
View File
@@ -23,6 +23,7 @@
* THE SOFTWARE.
******************************************************************************/
#include "d3d11_hooks.h"
#include "driver/dxgi/dxgi_wrapped.h"
#include "driver/ihv/amd/official/DXExt/AmdDxExtApi.h"
#include "hooks/hooks.h"
@@ -38,8 +39,6 @@
#endif
ID3D11Resource *UnwrapDXResource(void *dxObject);
ID3DDevice *GetD3D11DeviceIfAlloc(IUnknown *dev)
{
if(WrappedID3D11Device::IsAlloc(dev))
@@ -48,78 +47,9 @@ ID3DDevice *GetD3D11DeviceIfAlloc(IUnknown *dev)
return NULL;
}
enum NVAPI_DEVICE_FEATURE_LEVEL
{
dummy = 0,
};
typedef void *(__cdecl *PFNNVQueryInterface)(uint32_t ID);
typedef HRESULT (*PFNNVCreateDevice)(_In_opt_ IDXGIAdapter *, D3D_DRIVER_TYPE, HMODULE, UINT,
_In_reads_opt_(FeatureLevels) CONST D3D_FEATURE_LEVEL *,
UINT FeatureLevels, UINT, _COM_Outptr_opt_ ID3D11Device **,
_Out_opt_ D3D_FEATURE_LEVEL *,
_COM_Outptr_opt_ ID3D11DeviceContext **,
NVAPI_DEVICE_FEATURE_LEVEL *);
typedef HRESULT (*PFNNVCreateDeviceAndSwapChain)(
_In_opt_ IDXGIAdapter *, D3D_DRIVER_TYPE, HMODULE, UINT,
_In_reads_opt_(FeatureLevels) CONST D3D_FEATURE_LEVEL *, UINT FeatureLevels, UINT,
_In_opt_ CONST DXGI_SWAP_CHAIN_DESC *, _COM_Outptr_opt_ IDXGISwapChain **,
_COM_Outptr_opt_ ID3D11Device **, _Out_opt_ D3D_FEATURE_LEVEL *,
_COM_Outptr_opt_ ID3D11DeviceContext **, NVAPI_DEVICE_FEATURE_LEVEL *);
// this is the type of the lambda we use to route the call out to the 'real' function inside our
// generic wrapper.
// Could be any of D3D11CreateDevice, D3D11CreateDeviceAndSwapChain, or the nvapi equivalents
typedef std::function<HRESULT(IDXGIAdapter *, D3D_DRIVER_TYPE, HMODULE, UINT, CONST D3D_FEATURE_LEVEL *,
UINT FeatureLevels, UINT, CONST DXGI_SWAP_CHAIN_DESC *, IDXGISwapChain **,
ID3D11Device **, D3D_FEATURE_LEVEL *, ID3D11DeviceContext **)>
RealFunction;
#define NVENCAPI __stdcall
enum NVENCSTATUS
{
NV_ENC_SUCCESS = 0,
NV_ENC_ERR_INVALID_PTR = 6,
};
enum NV_ENC_INPUT_RESOURCE_TYPE
{
NV_ENC_INPUT_RESOURCE_TYPE_DIRECTX = 0x0,
NV_ENC_INPUT_RESOURCE_TYPE_CUDADEVICEPTR = 0x1,
NV_ENC_INPUT_RESOURCE_TYPE_CUDAARRAY = 0x2,
NV_ENC_INPUT_RESOURCE_TYPE_OPENGL_TEX = 0x3
};
struct NV_ENC_REGISTER_RESOURCE
{
uint32_t version;
NV_ENC_INPUT_RESOURCE_TYPE resourceType;
uint32_t dummy[4];
void *resourceToRegister;
// there is more data here but we don't allocate this structure only patch the above pointer, so
// we don't need it
};
typedef NVENCSTATUS(NVENCAPI *PNVENCREGISTERRESOURCE)(void *, NV_ENC_REGISTER_RESOURCE *params);
struct NV_ENCODE_API_FUNCTION_LIST
{
uint32_t version;
uint32_t reserved;
void *otherFunctions[30]; // other functions in the dispatch table
PNVENCREGISTERRESOURCE nvEncRegisterResource;
// there is more data here but we don't allocate this structure only patch the above pointer, so
// we don't need it
};
typedef NVENCSTATUS(NVENCAPI *PFN_NvEncodeAPICreateInstance)(NV_ENCODE_API_FUNCTION_LIST *);
class D3D11Hook : LibraryHook
{
public:
D3D11Hook() { m_InsideCreate = false; }
void RegisterHooks()
{
RDCLOG("Registering D3D11 hooks");
@@ -144,18 +74,6 @@ public:
LibraryHooks::RegisterLibraryHook(BIT_SPECIFIC_DLL("atidxx32.dll", "atidxx64.dll"), NULL);
AmdCreate11.Register(BIT_SPECIFIC_DLL("atidxx32.dll", "atidxx64.dll"), "AmdDxExtCreate11",
AmdCreate11_hook);
// nvapi has some seriously awful ""feature"" where it can wrap CreateDevice with some other
// extra parameter. Since we don't want to hook nvapi when it calls out to d3d11.dll, we have to
// intercept it here.
LibraryHooks::RegisterLibraryHook(BIT_SPECIFIC_DLL("nvapi.dll", "nvapi64.dll"), NULL);
nvapi_QueryInterface.Register(BIT_SPECIFIC_DLL("nvapi.dll", "nvapi64.dll"),
"nvapi_QueryInterface", nvapi_QueryInterface_hook);
// we need to wrap nvcodec to handle unwrapping D3D11 pointers passed to it
LibraryHooks::RegisterLibraryHook(BIT_SPECIFIC_DLL("nvEncodeAPI.dll", "nvEncodeAPI64.dll"), NULL);
NvEncodeCreate.Register(BIT_SPECIFIC_DLL("nvEncodeAPI.dll", "nvEncodeAPI64.dll"),
"NvEncodeAPICreateInstance", NvEncodeAPICreateInstance_hook);
}
private:
@@ -166,13 +84,22 @@ private:
// optional extension hooks
HookedFunction<PFNAmdDxExtCreate11> AmdCreate11;
HookedFunction<PFNNVQueryInterface> nvapi_QueryInterface;
HookedFunction<PFN_NvEncodeAPICreateInstance> NvEncodeCreate;
// re-entrancy detection (can happen in rare cases with e.g. fraps)
bool m_InsideCreate;
bool m_InsideCreate = false;
HRESULT Create_Internal(RealFunction real, __in_opt IDXGIAdapter *pAdapter,
friend HRESULT CreateD3D11_Internal(RealD3D11CreateFunction real, __in_opt IDXGIAdapter *pAdapter,
D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags,
__in_ecount_opt(FeatureLevels)
CONST D3D_FEATURE_LEVEL *pFeatureLevels,
UINT FeatureLevels, UINT SDKVersion,
__in_opt CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc,
__out_opt IDXGISwapChain **ppSwapChain,
__out_opt ID3D11Device **ppDevice,
__out_opt D3D_FEATURE_LEVEL *pFeatureLevel,
__out_opt ID3D11DeviceContext **ppImmediateContext);
HRESULT Create_Internal(RealD3D11CreateFunction real, __in_opt IDXGIAdapter *pAdapter,
D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags,
__in_ecount_opt(FeatureLevels) CONST D3D_FEATURE_LEVEL *pFeatureLevels,
UINT FeatureLevels, UINT SDKVersion,
@@ -335,153 +262,21 @@ private:
return E_FAIL;
}
PFNNVCreateDevice nvapi_CreateDevice_real = NULL;
PFNNVCreateDeviceAndSwapChain nvapi_CreateDeviceAndSwapChain_real = NULL;
static HRESULT WINAPI nvapi_CreateDevice(IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType,
HMODULE Software, UINT Flags,
CONST D3D_FEATURE_LEVEL *pFeatureLevels,
UINT FeatureLevels, UINT SDKVersion,
ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel,
ID3D11DeviceContext **ppImmediateContext,
NVAPI_DEVICE_FEATURE_LEVEL *outNVLevel)
{
return d3d11hooks.Create_Internal(
[=](IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags,
CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion,
CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, IDXGISwapChain **ppSwapChain,
ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel,
ID3D11DeviceContext **ppImmediateContext) {
// we know that when we come back in here the swapchain parameters will be NULL because
// that's what we pass below
RDCASSERT(!pSwapChainDesc && !ppSwapChain);
return d3d11hooks.nvapi_CreateDevice_real(
pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion,
ppDevice, pFeatureLevel, ppImmediateContext, outNVLevel);
},
pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, NULL,
NULL, ppDevice, pFeatureLevel, ppImmediateContext);
}
static HRESULT WINAPI nvapi_CreateDeviceAndSwapChain(
IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags,
CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion,
CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, IDXGISwapChain **ppSwapChain,
ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel,
ID3D11DeviceContext **ppImmediateContext, NVAPI_DEVICE_FEATURE_LEVEL *outNVLevel)
{
return d3d11hooks.Create_Internal(
[=](IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags,
CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion,
CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, IDXGISwapChain **ppSwapChain,
ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel,
ID3D11DeviceContext **ppImmediateContext) {
return d3d11hooks.nvapi_CreateDeviceAndSwapChain_real(
pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion,
pSwapChainDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext, outNVLevel);
},
pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion,
pSwapChainDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext);
}
static void *nvapi_QueryInterface_hook(uint32_t ID)
{
void *real = d3d11hooks.nvapi_QueryInterface()(ID);
if(real == NULL)
return real;
if(ID == 0x6A16D3A0)
{
d3d11hooks.nvapi_CreateDevice_real = (PFNNVCreateDevice)real;
return &nvapi_CreateDevice;
}
else if(ID == 0xBB939EE5)
{
d3d11hooks.nvapi_CreateDeviceAndSwapChain_real = (PFNNVCreateDeviceAndSwapChain)real;
return &nvapi_CreateDeviceAndSwapChain;
}
else if(ID == 0xAD298D3F || ID == 0x0150E828 || ID == 0x33C7358C || ID == 0x593E8644)
{
// these seem to be fetched inside NvAPI_Initialize so we allow them through to avoid
// causing problems
return real;
}
else
{
if(RenderDoc::Inst().IsVendorExtensionEnabled(VendorExtensions::NvAPI))
{
RDCDEBUG("NvAPI allowed: Returning %p for nvapi_QueryInterface(%x)", real, ID);
return real;
}
else
{
static int count = 0;
if(count < 10)
RDCWARN("NvAPI disabled: Returning NULL for nvapi_QueryInterface(%x)", ID);
count++;
return NULL;
}
}
}
PNVENCREGISTERRESOURCE real_nvEncRegisterResource = NULL;
static NVENCSTATUS NVENCAPI NvEncodeAPIRegisterResource_hook(void *encoder,
NV_ENC_REGISTER_RESOURCE *params)
{
if(!d3d11hooks.real_nvEncRegisterResource)
{
RDCERR("nvEncRegisterResource called without hooking NvEncodeAPICreateInstance!");
return NV_ENC_ERR_INVALID_PTR;
}
// only directx textures need to be unwrapped
if(!encoder || !params || params->resourceType != NV_ENC_INPUT_RESOURCE_TYPE_DIRECTX)
return d3d11hooks.real_nvEncRegisterResource(encoder, params);
// attempt to unwrap the handle in place
void *origHandle = params->resourceToRegister;
params->resourceToRegister = UnwrapDXResource(origHandle);
if(params->resourceToRegister == NULL)
{
RDCERR("Failed to unwrap DX handle %p, falling back to pass-through", origHandle);
params->resourceToRegister = origHandle;
}
// call out to the actual function
NVENCSTATUS ret = d3d11hooks.real_nvEncRegisterResource(encoder, params);
// restore the handle to the original value
params->resourceToRegister = origHandle;
return ret;
}
static NVENCSTATUS NVENCAPI NvEncodeAPICreateInstance_hook(NV_ENCODE_API_FUNCTION_LIST *functions)
{
NVENCSTATUS ret = d3d11hooks.NvEncodeCreate()(functions);
if(ret == NV_ENC_SUCCESS && functions && functions->nvEncRegisterResource)
{
// this is an encoded struct version, 7 is a magic value, 8.1 is the major.minor of nvcodec
// and 2 is the struct version
const uint32_t expectedVersion = 7 << 28 | 8 << 1 | 1 << 24 | 2 << 16;
if(functions->version != expectedVersion)
RDCWARN("Call to NvEncodeAPICreateInstance with version %x, expected %x",
functions->version, expectedVersion);
// we don't handle multiple different pointers coming back, but that seems unlikely.
RDCASSERT(d3d11hooks.real_nvEncRegisterResource == NULL ||
d3d11hooks.real_nvEncRegisterResource == functions->nvEncRegisterResource);
d3d11hooks.real_nvEncRegisterResource = functions->nvEncRegisterResource;
functions->nvEncRegisterResource = &NvEncodeAPIRegisterResource_hook;
}
return ret;
}
};
D3D11Hook D3D11Hook::d3d11hooks;
HRESULT CreateD3D11_Internal(RealD3D11CreateFunction real, __in_opt IDXGIAdapter *pAdapter,
D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags,
__in_ecount_opt(FeatureLevels) CONST D3D_FEATURE_LEVEL *pFeatureLevels,
UINT FeatureLevels, UINT SDKVersion,
__in_opt CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc,
__out_opt IDXGISwapChain **ppSwapChain,
__out_opt ID3D11Device **ppDevice,
__out_opt D3D_FEATURE_LEVEL *pFeatureLevel,
__out_opt ID3D11DeviceContext **ppImmediateContext)
{
return D3D11Hook::d3d11hooks.Create_Internal(
real, pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion,
pSwapChainDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext);
}
renderdoc/driver/d3d11/d3d11_hooks.h
+47
View File
@@ -0,0 +1,47 @@
/******************************************************************************
* The MIT License (MIT)
*
* Copyright (c) 2020 Baldur Karlsson
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
******************************************************************************/
#pragma once
#include <functional>
#include "driver/dx/official/d3d11.h"
#include "driver/dx/official/dxgi.h"
// this is the type of the lambda we use to route the call out to the 'real' function inside our
// generic wrapper.
// Could be any of D3D11CreateDevice, D3D11CreateDeviceAndSwapChain, or the nvapi equivalents
typedef std::function<HRESULT(IDXGIAdapter *, D3D_DRIVER_TYPE, HMODULE, UINT, CONST D3D_FEATURE_LEVEL *,
UINT FeatureLevels, UINT, CONST DXGI_SWAP_CHAIN_DESC *, IDXGISwapChain **,
ID3D11Device **, D3D_FEATURE_LEVEL *, ID3D11DeviceContext **)>
RealD3D11CreateFunction;
HRESULT CreateD3D11_Internal(RealD3D11CreateFunction real, __in_opt IDXGIAdapter *pAdapter,
D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags,
__in_ecount_opt(FeatureLevels) CONST D3D_FEATURE_LEVEL *pFeatureLevels,
UINT FeatureLevels, UINT SDKVersion,
__in_opt CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc,
__out_opt IDXGISwapChain **ppSwapChain,
__out_opt ID3D11Device **ppDevice,
__out_opt D3D_FEATURE_LEVEL *pFeatureLevel,
__out_opt ID3D11DeviceContext **ppImmediateContext);
renderdoc/driver/d3d11/renderdoc_d3d11.vcxproj
+1
View File
@@ -142,6 +142,7 @@
<ClInclude Include="d3d11_context.h" />
<ClInclude Include="d3d11_debug.h" />
<ClInclude Include="d3d11_device.h" />
<ClInclude Include="d3d11_hooks.h" />
<ClInclude Include="d3d11_manager.h" />
<ClInclude Include="d3d11_renderstate.h" />
<ClInclude Include="d3d11_rendertext.h" />
renderdoc/driver/d3d11/renderdoc_d3d11.vcxproj.filters
+1
View File
@@ -185,5 +185,6 @@
<ClInclude Include="d3d11_video.h">
<Filter>Wrapped</Filter>
</ClInclude>
<ClInclude Include="d3d11_hooks.h" />
</ItemGroup>
</Project>
renderdoc/driver/ihv/nv/NV.vcxproj
+3
View File
@@ -96,10 +96,13 @@
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="nvapi_hooks.cpp" />
<ClCompile Include="nv_counters.cpp" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="nv_counters.h" />
<ClInclude Include="official\nvapi\nvapi.h" />
<ClInclude Include="official\nvapi\nvapi_interface.h" />
<ClInclude Include="official\PerfKit\include\NvPmApi.h" />
</ItemGroup>
<ItemGroup>
renderdoc/driver/ihv/nv/NV.vcxproj.filters
+15
View File
@@ -7,6 +7,12 @@
<ClInclude Include="nv_counters.h">
<Filter>Counters</Filter>
</ClInclude>
<ClInclude Include="official\nvapi\nvapi.h">
<Filter>official\nvapi</Filter>
</ClInclude>
<ClInclude Include="official\nvapi\nvapi_interface.h">
<Filter>official\nvapi</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Filter Include="official">
@@ -21,10 +27,19 @@
<Filter Include="Counters">
<UniqueIdentifier>{630ba7bc-f1cd-4dd7-ba0d-5e4d689be9fd}</UniqueIdentifier>
</Filter>
<Filter Include="official\nvapi">
<UniqueIdentifier>{fb35dcda-304a-43af-abfd-86e81fb701ae}</UniqueIdentifier>
</Filter>
<Filter Include="nvapi">
<UniqueIdentifier>{f9b6bd25-7b71-4814-b8de-ca40545419a9}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="nv_counters.cpp">
<Filter>Counters</Filter>
</ClCompile>
<ClCompile Include="nvapi_hooks.cpp">
<Filter>nvapi</Filter>
</ClCompile>
</ItemGroup>
</Project>
renderdoc/driver/ihv/nv/nvapi_hooks.cpp
+291
View File
@@ -0,0 +1,291 @@
/******************************************************************************
* The MIT License (MIT)
*
* Copyright (c) 2020 Baldur Karlsson
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
******************************************************************************/
#include "driver/d3d11/d3d11_device.h"
#include "driver/d3d11/d3d11_hooks.h"
#include "hooks/hooks.h"
#include "official/nvapi/nvapi.h"
#include "official/nvapi/nvapi_interface.h"
#if ENABLED(RDOC_X64)
#define BIT_SPECIFIC_DLL(dll32, dll64) dll64
#else
#define BIT_SPECIFIC_DLL(dll32, dll64) dll32
#endif
ID3D11Resource *UnwrapDXResource(void *dxObject);
typedef void *(__cdecl *PFNNVQueryInterface)(uint32_t ID);
typedef HRESULT (*PFNNVCreateDeviceAndSwapChain)(
_In_opt_ IDXGIAdapter *, D3D_DRIVER_TYPE, HMODULE, UINT,
_In_reads_opt_(FeatureLevels) CONST D3D_FEATURE_LEVEL *, UINT FeatureLevels, UINT,
_In_opt_ CONST DXGI_SWAP_CHAIN_DESC *, _COM_Outptr_opt_ IDXGISwapChain **,
_COM_Outptr_opt_ ID3D11Device **, _Out_opt_ D3D_FEATURE_LEVEL *,
_COM_Outptr_opt_ ID3D11DeviceContext **, NVAPI_DEVICE_FEATURE_LEVEL *);
// this is the type of the lambda we use to route the call out to the 'real' function inside our
// generic wrapper.
// Could be any of D3D11CreateDevice, D3D11CreateDeviceAndSwapChain, or the nvapi equivalents
typedef std::function<HRESULT(IDXGIAdapter *, D3D_DRIVER_TYPE, HMODULE, UINT, CONST D3D_FEATURE_LEVEL *,
UINT FeatureLevels, UINT, CONST DXGI_SWAP_CHAIN_DESC *, IDXGISwapChain **,
ID3D11Device **, D3D_FEATURE_LEVEL *, ID3D11DeviceContext **)>
RealD3D11CreateFunction;
#define NVENCAPI __stdcall
enum NVENCSTATUS
{
NV_ENC_SUCCESS = 0,
NV_ENC_ERR_INVALID_PTR = 6,
};
enum NV_ENC_INPUT_RESOURCE_TYPE
{
NV_ENC_INPUT_RESOURCE_TYPE_DIRECTX = 0x0,
NV_ENC_INPUT_RESOURCE_TYPE_CUDADEVICEPTR = 0x1,
NV_ENC_INPUT_RESOURCE_TYPE_CUDAARRAY = 0x2,
NV_ENC_INPUT_RESOURCE_TYPE_OPENGL_TEX = 0x3
};
struct NV_ENC_REGISTER_RESOURCE
{
uint32_t version;
NV_ENC_INPUT_RESOURCE_TYPE resourceType;
uint32_t dummy[4];
void *resourceToRegister;
// there is more data here but we don't allocate this structure only patch the above pointer, so
// we don't need it
};
typedef NVENCSTATUS(NVENCAPI *PNVENCREGISTERRESOURCE)(void *, NV_ENC_REGISTER_RESOURCE *params);
struct NV_ENCODE_API_FUNCTION_LIST
{
uint32_t version;
uint32_t reserved;
void *otherFunctions[30]; // other functions in the dispatch table
PNVENCREGISTERRESOURCE nvEncRegisterResource;
// there is more data here but we don't allocate this structure only patch the above pointer, so
// we don't need it
};
typedef NVENCSTATUS(NVENCAPI *PFN_NvEncodeAPICreateInstance)(NV_ENCODE_API_FUNCTION_LIST *);
class NVHook : LibraryHook
{
public:
void RegisterHooks()
{
RDCLOG("Registering nvidia hooks");
LibraryHooks::RegisterLibraryHook(BIT_SPECIFIC_DLL("nvapi.dll", "nvapi64.dll"), NULL);
nvapi_QueryInterface.Register(BIT_SPECIFIC_DLL("nvapi.dll", "nvapi64.dll"),
"nvapi_QueryInterface", nvapi_QueryInterface_hook);
// we need to wrap nvcodec to handle unwrapping D3D11 pointers passed to it
LibraryHooks::RegisterLibraryHook(BIT_SPECIFIC_DLL("nvEncodeAPI.dll", "nvEncodeAPI64.dll"), NULL);
NvEncodeCreate.Register(BIT_SPECIFIC_DLL("nvEncodeAPI.dll", "nvEncodeAPI64.dll"),
"NvEncodeAPICreateInstance", NvEncodeAPICreateInstance_hook);
// we need ID -> function lookup, not function -> ID as the table gives us. We also want fairly
// quick lookup since some programs call nvapi_QueryInterface at *high* frequency.
for(const NVAPI_INTERFACE_TABLE &iface : nvapi_interface_table)
nvapi_lookup[iface.id] = iface.func;
}
private:
static NVHook nvhooks;
std::unordered_map<uint32_t, const char *> nvapi_lookup;
HookedFunction<PFNNVQueryInterface> nvapi_QueryInterface;
HookedFunction<PFN_NvEncodeAPICreateInstance> NvEncodeCreate;
#define HOOK_NVAPI(fname, ID) HookedFunction<decltype(&fname)> fname;
#define WHITELIST_NVAPI(fname, ID)
#define NVAPI_FUNCS() \
HOOK_NVAPI(NvAPI_D3D11_CreateDevice, 0x6A16D3A0); \
HOOK_NVAPI(NvAPI_D3D11_CreateDeviceAndSwapChain, 0xBB939EE5); \
WHITELIST_NVAPI(NvAPI_Initialize, 0x0150E828);
NVAPI_FUNCS();
static HRESULT WINAPI NvAPI_D3D11_CreateDevice_hook(
IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags,
CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion,
ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel,
ID3D11DeviceContext **ppImmediateContext, NVAPI_DEVICE_FEATURE_LEVEL *outNVLevel)
{
return CreateD3D11_Internal(
[=](IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags,
CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion,
CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, IDXGISwapChain **ppSwapChain,
ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel,
ID3D11DeviceContext **ppImmediateContext) {
// we know that when we come back in here the swapchain parameters will be NULL because
// that's what we pass below
RDCASSERT(!pSwapChainDesc && !ppSwapChain);
return nvhooks.NvAPI_D3D11_CreateDevice()(
pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion,
ppDevice, pFeatureLevel, ppImmediateContext, outNVLevel);
},
pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion, NULL,
NULL, ppDevice, pFeatureLevel, ppImmediateContext);
}
static HRESULT WINAPI NvAPI_D3D11_CreateDeviceAndSwapChain_hook(
IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags,
CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion,
CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, IDXGISwapChain **ppSwapChain,
ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel,
ID3D11DeviceContext **ppImmediateContext, NVAPI_DEVICE_FEATURE_LEVEL *outNVLevel)
{
return CreateD3D11_Internal(
[=](IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags,
CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels, UINT SDKVersion,
CONST DXGI_SWAP_CHAIN_DESC *pSwapChainDesc, IDXGISwapChain **ppSwapChain,
ID3D11Device **ppDevice, D3D_FEATURE_LEVEL *pFeatureLevel,
ID3D11DeviceContext **ppImmediateContext) {
return nvhooks.NvAPI_D3D11_CreateDeviceAndSwapChain()(
pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion,
pSwapChainDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext, outNVLevel);
},
pAdapter, DriverType, Software, Flags, pFeatureLevels, FeatureLevels, SDKVersion,
pSwapChainDesc, ppSwapChain, ppDevice, pFeatureLevel, ppImmediateContext);
}
static void *nvapi_QueryInterface_hook(uint32_t ID)
{
void *real = nvhooks.nvapi_QueryInterface()(ID);
if(real == NULL)
return real;
#undef HOOK_NVAPI
#define HOOK_NVAPI(fname, ID) \
case ID: \
{ \
nvhooks.fname.SetFuncPtr(real); \
return &CONCAT(fname, _hook); \
}
#undef WHITELIST_NVAPI
#define WHITELIST_NVAPI(fname, ID) \
case ID: return real;
switch(ID)
{
NVAPI_FUNCS();
// unknown, but these are fetched inside NvAPI_Initialize so allow them through to avoid
// causing problems.
WHITELIST_NVAPI(unknown_func, 0xAD298D3F);
WHITELIST_NVAPI(unknown_func, 0x33C7358C);
WHITELIST_NVAPI(unknown_func, 0x593E8644);
default: break;
}
const char *cname = nvhooks.nvapi_lookup[ID];
rdcstr name = cname ? cname : StringFormat::Fmt("0x%x", ID);
if(RenderDoc::Inst().IsVendorExtensionEnabled(VendorExtensions::NvAPI))
{
RDCDEBUG("NvAPI allowed: Returning %p for nvapi_QueryInterface(%s)", real, name.c_str());
return real;
}
else
{
static int count = 0;
if(count < 10)
RDCWARN("NvAPI disabled: Returning NULL for nvapi_QueryInterface(%s)", name.c_str());
count++;
return NULL;
}
}
PNVENCREGISTERRESOURCE real_nvEncRegisterResource = NULL;
static NVENCSTATUS NVENCAPI NvEncodeAPIRegisterResource_hook(void *encoder,
NV_ENC_REGISTER_RESOURCE *params)
{
if(!nvhooks.real_nvEncRegisterResource)
{
RDCERR("nvEncRegisterResource called without hooking NvEncodeAPICreateInstance!");
return NV_ENC_ERR_INVALID_PTR;
}
// only directx textures need to be unwrapped
if(!encoder || !params || params->resourceType != NV_ENC_INPUT_RESOURCE_TYPE_DIRECTX)
return nvhooks.real_nvEncRegisterResource(encoder, params);
// attempt to unwrap the handle in place
void *origHandle = params->resourceToRegister;
params->resourceToRegister = UnwrapDXResource(origHandle);
if(params->resourceToRegister == NULL)
{
RDCERR("Failed to unwrap DX handle %p, falling back to pass-through", origHandle);
params->resourceToRegister = origHandle;
}
// call out to the actual function
NVENCSTATUS ret = nvhooks.real_nvEncRegisterResource(encoder, params);
// restore the handle to the original value
params->resourceToRegister = origHandle;
return ret;
}
static NVENCSTATUS NVENCAPI NvEncodeAPICreateInstance_hook(NV_ENCODE_API_FUNCTION_LIST *functions)
{
NVENCSTATUS ret = nvhooks.NvEncodeCreate()(functions);
if(ret == NV_ENC_SUCCESS && functions && functions->nvEncRegisterResource)
{
// this is an encoded struct version, 7 is a magic value, 8.1 is the major.minor of nvcodec
// and 2 is the struct version
const uint32_t expectedVersion = 7 << 28 | 8 << 1 | 1 << 24 | 2 << 16;
if(functions->version != expectedVersion)
RDCWARN("Call to NvEncodeAPICreateInstance with version %x, expected %x",
functions->version, expectedVersion);
// we don't handle multiple different pointers coming back, but that seems unlikely.
RDCASSERT(nvhooks.real_nvEncRegisterResource == NULL ||
nvhooks.real_nvEncRegisterResource == functions->nvEncRegisterResource);
nvhooks.real_nvEncRegisterResource = functions->nvEncRegisterResource;
functions->nvEncRegisterResource = &NvEncodeAPIRegisterResource_hook;
}
return ret;
}
};
NVHook NVHook::nvhooks;