Only apply D3D12Core.dll signature check to untrusted files

* UE5 at least currently (as of this commit) re-signs D3D12Core.dll by Epic.
  Rather than failing to load all such captures, only fail if the capture is
  marked as downloaded since local captures should be implicitly trusted.

renderdoc/driver/d3d12/d3d12_common.h

@@ -59,8 +59,8 @@ struct D3D12MarkerRegion
 bool EnableD3D12DebugLayer(PFN_D3D12_GET_DEBUG_INTERFACE getDebugInterface = NULL);
 HRESULT EnumAdapterByLuid(IDXGIFactory1 *factory, LUID luid, IDXGIAdapter **pAdapter);
 
-void D3D12_PrepareReplaySDKVersion(UINT SDKVersion, bytebuf d3d12core, bytebuf d3d12sdklayers,
-                                   HMODULE d3d12lib);
+void D3D12_PrepareReplaySDKVersion(bool untrustedCapture, UINT SDKVersion, bytebuf d3d12core,
+                                   bytebuf d3d12sdklayers, HMODULE d3d12lib);
 void D3D12_CleanupReplaySDK();
 
 inline void SetObjName(ID3D12Object *obj, const rdcstr &utf8name)

renderdoc/driver/d3d12/d3d12_replay.cpp

@@ -4382,7 +4382,8 @@ RDResult D3D12_CreateReplayDevice(RDCFile *rdc, const ReplayOptions &opts, IRepl
   if(initParams.MinimumFeatureLevel < D3D_FEATURE_LEVEL_11_0)
     initParams.MinimumFeatureLevel = D3D_FEATURE_LEVEL_11_0;
 
-  D3D12_PrepareReplaySDKVersion(initParams.SDKVersion, D3D12Core, D3D12SDKLayers, D3D12Lib);
+  D3D12_PrepareReplaySDKVersion(rdc && rdc->IsUntrusted(), initParams.SDKVersion, D3D12Core,
+                                D3D12SDKLayers, D3D12Lib);
 
   const bool isProxy = (rdc == NULL);
 

renderdoc/driver/d3d12/d3d12_sdk_select.cpp

@@ -26,6 +26,7 @@
 #include "core/settings.h"
 #include "hooks/hooks.h"
 #include "strings/string_utils.h"
+#include "tinyfiledialogs/tinyfiledialogs.h"
 #include "d3d12_common.h"
 
 RDOC_CONFIG(rdcstr, D3D12_D3D12CoreDirPath, "",
@@ -158,7 +159,7 @@ HRESULT WINAPI Hooked_SDKLayers_D3D12GetInterface(_In_ REFCLSID rclsid, _In_ REF
 }
 
 // Can check signatures of .exe and .dll files
-bool IsSignedByMicrosoft(const rdcstr &filename)
+bool IsSignedByMicrosoft(const rdcstr &filename, rdcstr &signer)
 {
   bool IsSignatureValid = false;
 
@@ -188,7 +189,7 @@ bool IsSignedByMicrosoft(const rdcstr &filename)
   DWORD signer_info_size = 0;
   PCCERT_CONTEXT cert_context = NULL;
   CERT_INFO cert_info = {};
-  LPTSTR signer_name = NULL;
+  LPWSTR signer_name = NULL;
   DWORD signer_name_length = 0;
 
   do
@@ -269,7 +270,7 @@ bool IsSignedByMicrosoft(const rdcstr &filename)
     }
 
     // Allocate memory for subject name.
-    signer_name = (LPTSTR)LocalAlloc(LPTR, signer_name_length * sizeof(WCHAR));
+    signer_name = (LPWSTR)LocalAlloc(LPTR, signer_name_length * sizeof(WCHAR));
     if(!signer_name)
     {
       break;
@@ -282,6 +283,8 @@ bool IsSignedByMicrosoft(const rdcstr &filename)
       break;
     }
 
+    signer = StringFormat::Wide2UTF8(rdcwstr(signer_name));
+
     // Check whether signer is Microsoft
     // Since Microsoft uses multiple different signatures,
     // We just check whether "Microsoft" is a substring of signer simple name
@@ -311,7 +314,7 @@ bool IsSignedByMicrosoft(const rdcstr &filename)
   return IsSignatureValid;
 }
 
-void D3D12_PrepareReplaySDKVersion(UINT SDKVersion, bytebuf d3d12core_file,
+void D3D12_PrepareReplaySDKVersion(bool untrustedCapture, UINT SDKVersion, bytebuf d3d12core_file,
                                    bytebuf d3d12sdklayers_file, HMODULE d3d12lib)
 {
   // D3D12Core shouldn't be loaded at this point, but it might be due to bugs. If it is, we can't do
@@ -445,7 +448,10 @@ void D3D12_PrepareReplaySDKVersion(UINT SDKVersion, bytebuf d3d12core_file,
         FileIO::fwrite(d3d12core_file.data(), 1, d3d12core_file.size(), f);
         FileIO::fclose(f);
 
-        if(!IsSignedByMicrosoft(filename))
+        rdcstr signer;
+        // trusted captures (i.e. not marked as downloaded from the internet by windows) will skip
+        // this check entirely. Untrusted captures will verify the DLL signature is Microsoft signed
+        if(untrustedCapture && !IsSignedByMicrosoft(filename, signer))
         {
           if(D3D12_Debug_IgnoreSignatureCheck())
           {
@@ -455,12 +461,34 @@ void D3D12_PrepareReplaySDKVersion(UINT SDKVersion, bytebuf d3d12core_file,
           }
           else
           {
-            FileIO::Delete(filename);
-            RDCERR(
-                "Can't verify the digital signature of the D3D12Core.dll embedded in capture, it "
-                "won't be loaded. If the capture came from a trusted source and you want to load "
-                "unsigned dll's, set D3D12.Debug.IgnoreSignatureCheck to true");
-            break;
+            RDCLOG("D3D12Core signed by '%s' instead of MS", signer.c_str());
+
+            rdcstr msg =
+                "Capture file contains an embedded D3D12 dll which is not correctly signed by "
+                "Microsoft.\n\n";
+            if(signer.empty())
+              msg += "There is no signature at all.\n\n";
+            else
+              msg += "The file is signed by '" + signer + "'.\n\n";
+
+            msg +=
+                "If you want to load the capture anyway, click yes. To use the system version of "
+                "D3D12 click no.";
+
+            int res = tinyfd_messageBox("Unexpected DLL signature", msg.c_str(), "yesnocancel",
+                                        "error", 2);
+            // 1 == yes, either no or cancel will abort the load
+            if(res != 1)
+            {
+              FileIO::Delete(filename);
+              RDCERR(
+                  "Can't verify the digital signature of the D3D12Core.dll embedded in capture, it "
+                  "won't be loaded. If the capture came from a trusted source and you want to load "
+                  "unsigned dll's, set D3D12.Debug.IgnoreSignatureCheck to true");
+              break;
+            }
+
+            RDCLOG("User selected to continue with load.");
           }
         }
 

renderdoc/os/os_specific.h

@@ -316,6 +316,8 @@ FILE *fopen(const rdcstr &filename, FileMode mode);
 size_t fread(void *buf, size_t elementSize, size_t count, FILE *f);
 size_t fwrite(const void *buf, size_t elementSize, size_t count, FILE *f);
 
+bool IsUntrustedFile(const rdcstr &filename);
+
 bool exists(const rdcstr &filename);
 
 rdcstr ErrorString();

renderdoc/os/posix/posix_stringio.cpp

@@ -451,6 +451,12 @@ int fclose(FILE *f)
   return ::fclose(f);
 }
 
+bool IsUntrustedFile(const rdcstr &filename)
+{
+  // do android/linux have any way of marking files as potentially unsafe?
+  return false;
+}
+
 bool exists(const rdcstr &filename)
 {
   struct ::stat st;

renderdoc/os/win32/win32_stringio.cpp

@@ -30,6 +30,7 @@
 #include <string.h>
 #include <tchar.h>
 #include <time.h>
+#include <urlmon.h>
 #include <set>
 #include "api/app/renderdoc_app.h"
 #include "api/replay/data_types.h"
@@ -604,6 +605,45 @@ FILE *fopen(const rdcstr &filename, FileMode mode)
   return ret;
 }
 
+bool IsUntrustedFile(const rdcstr &filename)
+{
+  IPersistFile *file = NULL;
+  HRESULT hr = CoCreateInstance(CLSID_PersistentZoneIdentifier, 0, CLSCTX_INPROC_SERVER,
+                                __uuidof(IPersistFile), (void **)&file);
+  // we default to trusted on failure
+  if(FAILED(hr))
+  {
+    SAFE_RELEASE(file);
+    return false;
+  }
+  rdcwstr wfn = StringFormat::UTF82Wide(filename);
+  for(size_t i = 0; i < wfn.length(); i++)
+    if(wfn[i] == L'/')
+      wfn[i] = L'\\';
+  hr = file->Load(wfn.c_str(), STGM_READ);
+  if(FAILED(hr))
+  {
+    SAFE_RELEASE(file);
+    return false;
+  }
+  IZoneIdentifier *zone = NULL;
+  hr = file->QueryInterface(__uuidof(IZoneIdentifier), (void **)&zone);
+  if(FAILED(hr))
+  {
+    SAFE_RELEASE(file);
+    SAFE_RELEASE(zone);
+    return false;
+  }
+  DWORD zoneValue = URLZONE_LOCAL_MACHINE;
+  hr = zone->GetId(&zoneValue);
+  if(FAILED(hr))
+    zoneValue = URLZONE_LOCAL_MACHINE;
+  SAFE_RELEASE(file);
+  SAFE_RELEASE(zone);
+  // internet and worse are considered untrusted
+  return zoneValue >= URLZONE_INTERNET;
+}
+
 bool exists(const rdcstr &filename)
 {
   rdcwstr wfn = StringFormat::UTF82Wide(filename);

renderdoc/serialise/rdcfile.cpp

@@ -249,6 +249,8 @@ void RDCFile::Open(const rdcstr &path)
 
   RDCLOG("Opening RDCFile %s", path.c_str());
 
+  m_Untrusted = FileIO::IsUntrustedFile(path);
+
   // ensure section header is compiled correctly
   RDCCOMPILE_ASSERT(offsetof(BinarySectionHeader, name) == sizeof(uint32_t) * 10,
                     "BinarySectionHeader size has changed or contains padding");

renderdoc/serialise/rdcfile.h

@@ -74,6 +74,7 @@ public:
   // creates a new file with current properties, file will be overwritten if it already exists
   void Create(const rdcstr &filename);
 
+  bool IsUntrusted() const { return m_Untrusted; }
   const RDResult &Error() const { return m_Error; }
   RDCDriver GetDriver() const { return m_Driver; }
   const rdcstr &GetDriverName() const { return m_DriverName; }
@@ -110,6 +111,7 @@ private:
   double m_TimeFrequency = 1.0;
   RDCThumb m_Thumb;
 
+  bool m_Untrusted = false;
   RDResult m_Error;
 
   struct SectionLocation