When grabbing D3D12Core also grab D3D12SDKLayers for validation

* The D3D12SDKLayers needs to match the D3D12Core, so we save both when possible
  (the SDKLayers dll might not be loaded during capture but it should be next to
  the D3D12Core. On replay if we have one we put it next to the D3D12Core we're
  using for replay, and if not we forcibly disable debug otherwise we may crash.

renderdoc/api/replay/renderdoc_tostr.inl

@@ -1030,6 +1030,7 @@ rdcstr DoStringise(const SectionType &el)
     STRINGISE_ENUM_CLASS_NAMED(EmbeddedLogfile, "renderdoc/internal/logfile");
     STRINGISE_ENUM_CLASS_NAMED(EditedShaders, "renderdoc/ui/edits");
     STRINGISE_ENUM_CLASS_NAMED(D3D12Core, "renderdoc/internal/d3d12core");
+    STRINGISE_ENUM_CLASS_NAMED(D3D12SDKLayers, "renderdoc/internal/d3d12sdklayers");
   }
   END_ENUM_STRINGISE();
 }

renderdoc/api/replay/replay_enums.h

@@ -104,6 +104,12 @@ version of RenderDoc that addes a new section type. They should be considered eq
   This section contains an internal copy of D3D12Core for replaying.
 
   The name for this section will be "renderdoc/internal/d3d12core".
+
+.. data:: D3D12SDKLayers
+
+  This section contains an internal copy of D3D12SDKLayers for replaying.
+
+  The name for this section will be "renderdoc/internal/d3d12sdklayers".
 )");
 enum class SectionType : uint32_t
 {
@@ -119,6 +125,7 @@ enum class SectionType : uint32_t
   EmbeddedLogfile,
   EditedShaders,
   D3D12Core,
+  D3D12SDKLayers,
   Count,
 };
 

renderdoc/driver/d3d12/d3d12_common.h

@@ -59,7 +59,8 @@ struct D3D12MarkerRegion
 bool EnableD3D12DebugLayer(PFN_D3D12_GET_DEBUG_INTERFACE getDebugInterface = NULL);
 HRESULT EnumAdapterByLuid(IDXGIFactory1 *factory, LUID luid, IDXGIAdapter **pAdapter);
 
-void D3D12_PrepareReplaySDKVersion(UINT SDKVersion, bytebuf d3d12core, HMODULE d3d12lib);
+void D3D12_PrepareReplaySDKVersion(UINT SDKVersion, bytebuf d3d12core, bytebuf d3d12sdklayers,
+                                   HMODULE d3d12lib);
 void D3D12_CleanupReplaySDK();
 
 inline void SetObjName(ID3D12Object *obj, const rdcstr &utf8name)

renderdoc/driver/d3d12/d3d12_device.cpp

@@ -2601,24 +2601,52 @@ bool WrappedID3D12Device::EndFrameCapture(void *dev, void *wnd)
   {
     if(rdc)
     {
-      wchar_t core_filename[MAX_PATH + 1] = {};
-      GetModuleFileNameW(D3D12Core, core_filename, MAX_PATH);
+      wchar_t wide_core_filename[MAX_PATH + 1] = {};
+      GetModuleFileNameW(D3D12Core, wide_core_filename, MAX_PATH);
 
-      bytebuf core_file;
-      FileIO::ReadAll(StringFormat::Wide2UTF8(core_filename), core_file);
+      rdcstr core_filename = StringFormat::Wide2UTF8(wide_core_filename);
 
-      SectionProperties props;
+      bytebuf buf;
+      FileIO::ReadAll(core_filename, buf);
 
-      props.flags = SectionFlags::ZstdCompressed;
-      props.version = 1;
-      props.type = SectionType::D3D12Core;
+      {
+        SectionProperties props;
 
-      captureWriter = rdc->WriteSection(props);
+        props.flags = SectionFlags::ZstdCompressed;
+        props.version = 1;
+        props.type = SectionType::D3D12Core;
 
-      captureWriter->Write(core_file.data(), core_file.size());
+        captureWriter = rdc->WriteSection(props);
 
-      captureWriter->Finish();
-      SAFE_DELETE(captureWriter);
+        captureWriter->Write(buf.data(), buf.size());
+
+        captureWriter->Finish();
+        SAFE_DELETE(captureWriter);
+      }
+
+      buf.clear();
+
+      // try to grab the SDK layers which will be needed for debug on replay
+
+      rdcstr sdklayers_filename = get_dirname(core_filename) + "/d3d12sdklayers.dll";
+
+      FileIO::ReadAll(sdklayers_filename, buf);
+
+      if(!buf.empty())
+      {
+        SectionProperties props;
+
+        props.flags = SectionFlags::ZstdCompressed;
+        props.version = 1;
+        props.type = SectionType::D3D12SDKLayers;
+
+        captureWriter = rdc->WriteSection(props);
+
+        captureWriter->Write(buf.data(), buf.size());
+
+        captureWriter->Finish();
+        SAFE_DELETE(captureWriter);
+      }
     }
 
     FreeLibrary(D3D12Core);

renderdoc/driver/d3d12/d3d12_replay.cpp

@@ -3922,7 +3922,7 @@ ReplayStatus D3D12_CreateReplayDevice(RDCFile *rdc, const ReplayOptions &opts, I
   }
 
   D3D12InitParams initParams;
-  bytebuf D3D12Core;
+  bytebuf D3D12Core, D3D12SDKLayers;
 
   uint64_t ver = D3D12InitParams::CurrentVersion;
 
@@ -3980,6 +3980,26 @@ ReplayStatus D3D12_CreateReplayDevice(RDCFile *rdc, const ReplayOptions &opts, I
       D3D12Core.resize((size_t)props.uncompressedSize);
       reader->Read(D3D12Core.data(), props.uncompressedSize);
 
+      RDCLOG("Got D3D12Core.dll of size %llu (decompressed from %llu)", props.uncompressedSize,
+             props.compressedSize);
+
+      RDCASSERT(reader->AtEnd());
+      delete reader;
+    }
+
+    sectionIdx = rdc->SectionIndex(SectionType::D3D12SDKLayers);
+
+    if(sectionIdx >= 0)
+    {
+      SectionProperties props = rdc->GetSectionProperties(sectionIdx);
+      reader = rdc->ReadSection(sectionIdx);
+
+      D3D12SDKLayers.resize((size_t)props.uncompressedSize);
+      reader->Read(D3D12SDKLayers.data(), props.uncompressedSize);
+
+      RDCLOG("Got D3D12SDKLayers.dll of size %llu (decompressed from %llu)", props.uncompressedSize,
+             props.compressedSize);
+
       RDCASSERT(reader->AtEnd());
       delete reader;
     }
@@ -3988,7 +4008,7 @@ ReplayStatus D3D12_CreateReplayDevice(RDCFile *rdc, const ReplayOptions &opts, I
   if(initParams.MinimumFeatureLevel < D3D_FEATURE_LEVEL_11_0)
     initParams.MinimumFeatureLevel = D3D_FEATURE_LEVEL_11_0;
 
-  D3D12_PrepareReplaySDKVersion(initParams.SDKVersion, D3D12Core, D3D12Lib);
+  D3D12_PrepareReplaySDKVersion(initParams.SDKVersion, D3D12Core, D3D12SDKLayers, D3D12Lib);
 
   if(rdc)
   {
@@ -4085,6 +4105,14 @@ ReplayStatus D3D12_CreateReplayDevice(RDCFile *rdc, const ReplayOptions &opts, I
 
   bool shouldEnableDebugLayer = opts.apiValidation;
 
+  if(shouldEnableDebugLayer && !D3D12Core.empty() && D3D12SDKLayers.empty())
+  {
+    RDCWARN(
+        "Not enabling D3D debug layers because we captured a D3D12Core.dll but not a matching "
+        "D3D12SDKLayers.dll, so this may crash");
+    shouldEnableDebugLayer = false;
+  }
+
   if(shouldEnableDebugLayer)
   {
     debugLayerEnabled = EnableD3D12DebugLayer();

renderdoc/driver/d3d12/d3d12_sdk_select.cpp

@@ -27,7 +27,7 @@
 #include "strings/string_utils.h"
 #include "d3d12_common.h"
 
-RDOC_CONFIG(rdcstr, D3D12_D3D12CorePath, "",
+RDOC_CONFIG(rdcstr, D3D12_D3D12CoreDirPath, "",
             "The location of the D3D12Core library. This path should be the directory that "
             "contains the D3D12Core.dll that you want to use.");
 
@@ -97,36 +97,43 @@ struct WrappedCoreModule : public ID3D12CoreModule
 };
 
 using PFN_D3D12_GET_INTERFACE = decltype(&D3D12GetInterface);
-HookedFunction<PFN_D3D12_GET_INTERFACE> D3D12GetInterface_hook;
+HookedFunction<PFN_D3D12_GET_INTERFACE> D3D12GetInterface_Core_hook;
+HookedFunction<PFN_D3D12_GET_INTERFACE> D3D12GetInterface_SDKLayers_hook;
 
 HMODULE Hooked_D3D12LoadLibrary(const rdcstr &filename, HANDLE h, DWORD flags)
 {
-  // if we detect the intercepted call to D3D12Core.dll and we have a redirect path, load that
-  // redirect path instead
-  if(strlower(filename).contains("d3d12core.dll") && !D3D12Core_Override_Path.empty())
+  if(!D3D12Core_Override_Path.empty())
   {
-    HMODULE ret =
-        LoadLibraryExW(StringFormat::UTF82Wide(D3D12Core_Override_Path).c_str(), NULL, flags);
-
-    if(ret)
+    // if we detect the intercepted call to a D3D12 dll and we have a redirect path, load that
+    // redirect path instead
+    for(const rdcstr &dll : {"d3d12core.dll", "d3d12sdklayers.dll"})
     {
-      Win32_ManualHookModule("d3d12core.dll", ret);
-    }
-    else
-    {
-      RDCERR("Error loading D3D12Core.dll from %s", D3D12Core_Override_Path.c_str());
-    }
+      if(strlower(filename).contains(dll))
+      {
+        HMODULE ret = LoadLibraryExW(
+            StringFormat::UTF82Wide(D3D12Core_Override_Path + "/" + dll).c_str(), NULL, flags);
 
-    return ret;
+        if(ret)
+        {
+          Win32_ManualHookModule(dll, ret);
+        }
+        else
+        {
+          RDCERR("Error loading %s from %s", dll.c_str(), D3D12Core_Override_Path.c_str());
+        }
+
+        return ret;
+      }
+    }
   }
 
   return NULL;
 }
 
-HRESULT WINAPI Hooked_D3D12GetInterface(_In_ REFCLSID rclsid, _In_ REFIID riid,
-                                        _COM_Outptr_opt_ void **ppvDebug)
+HRESULT WINAPI Hooked_Core_D3D12GetInterface(_In_ REFCLSID rclsid, _In_ REFIID riid,
+                                             _COM_Outptr_opt_ void **ppvDebug)
 {
-  HRESULT ret = D3D12GetInterface_hook()(rclsid, riid, ppvDebug);
+  HRESULT ret = D3D12GetInterface_Core_hook()(rclsid, riid, ppvDebug);
 
   // intercept the interface with our own wrapper to ensure version checking is silenced
   if(SUCCEEDED(ret) && riid == __uuidof(ID3D12CoreModule))
@@ -135,7 +142,20 @@ HRESULT WINAPI Hooked_D3D12GetInterface(_In_ REFCLSID rclsid, _In_ REFIID riid,
   return ret;
 }
 
-void D3D12_PrepareReplaySDKVersion(UINT SDKVersion, bytebuf d3d12core_file, HMODULE d3d12lib)
+HRESULT WINAPI Hooked_SDKLayers_D3D12GetInterface(_In_ REFCLSID rclsid, _In_ REFIID riid,
+                                                  _COM_Outptr_opt_ void **ppvDebug)
+{
+  HRESULT ret = D3D12GetInterface_SDKLayers_hook()(rclsid, riid, ppvDebug);
+
+  // intercept the interface with our own wrapper to ensure version checking is silenced
+  if(SUCCEEDED(ret) && riid == __uuidof(ID3D12CoreModule))
+    *ppvDebug = (ID3D12CoreModule *)(new WrappedCoreModule((ID3D12CoreModule *)*ppvDebug));
+
+  return ret;
+}
+
+void D3D12_PrepareReplaySDKVersion(UINT SDKVersion, bytebuf d3d12core_file,
+                                   bytebuf d3d12sdklayers_file, HMODULE d3d12lib)
 {
   // D3D12Core shouldn't be loaded at this point, but it might be due to bugs. If it is, we can't do
   // anything to change it anymore so we have to just handle what we have
@@ -225,7 +245,10 @@ void D3D12_PrepareReplaySDKVersion(UINT SDKVersion, bytebuf d3d12core_file, HMOD
 
     Win32_RegisterManualModuleHooking();
 
-    D3D12GetInterface_hook.Register("d3d12core.dll", "D3D12GetInterface", Hooked_D3D12GetInterface);
+    D3D12GetInterface_Core_hook.Register("d3d12core.dll", "D3D12GetInterface",
+                                         Hooked_Core_D3D12GetInterface);
+    D3D12GetInterface_SDKLayers_hook.Register("d3d12sdklayers.dll", "D3D12GetInterface",
+                                              Hooked_SDKLayers_D3D12GetInterface);
 
     Win32_InterceptLibraryLoads(Hooked_D3D12LoadLibrary);
   }
@@ -235,7 +258,7 @@ void D3D12_PrepareReplaySDKVersion(UINT SDKVersion, bytebuf d3d12core_file, HMOD
   Win32_ManualHookModule("d3d12.dll", d3d12lib);
 
   // *always* use the user's path if it exists
-  D3D12Core_Override_Path = D3D12_D3D12CorePath();
+  D3D12Core_Override_Path = D3D12_D3D12CoreDirPath();
 
   if(D3D12Core_Override_Path.empty() || !FileIO::exists(D3D12Core_Override_Path.c_str()))
   {
@@ -265,8 +288,20 @@ void D3D12_PrepareReplaySDKVersion(UINT SDKVersion, bytebuf d3d12core_file, HMOD
         FileIO::fwrite(d3d12core_file.data(), 1, d3d12core_file.size(), f);
         FileIO::fclose(f);
 
-        D3D12Core_Override_Path = filename;
-        D3D12Core_Temp_Path = filename;
+        D3D12Core_Override_Path = get_dirname(filename);
+        D3D12Core_Temp_Path = get_dirname(filename);
+
+        filename = get_dirname(filename) + "/d3d12sdklayers.dll";
+
+        f = FileIO::fopen(filename.c_str(), FileIO::WriteBinary);
+
+        // if we can write to this file, we have exclusive use of it so let's write it and use it
+        if(f)
+        {
+          FileIO::fwrite(d3d12sdklayers_file.data(), 1, d3d12sdklayers_file.size(), f);
+          FileIO::fclose(f);
+        }
+
         break;
       }
     }
@@ -282,6 +317,7 @@ void D3D12_CleanupReplaySDK()
 {
   if(!D3D12Core_Temp_Path.empty() && FileIO::exists(D3D12Core_Temp_Path.c_str()))
   {
-    FileIO::Delete(D3D12Core_Temp_Path.c_str());
+    FileIO::Delete((D3D12Core_Temp_Path + "/d3d12core.dll").c_str());
+    FileIO::Delete((D3D12Core_Temp_Path + "/d3d12sdklayers.dll").c_str());
   }
 }