Add partial OpenSharedHandle support

Add partial (as in write only, no initial data) D3D12 OpenSharedHandle support

renderdoc/driver/d3d12/d3d12_common.h

@@ -794,5 +794,6 @@ enum class D3D12Chunk : uint32_t
   List_SetSamplePositions,
   List_SetViewInstanceMask,
   List_WriteBufferImmediate,
+  Device_OpenSharedHandle,
   Max,
 };

renderdoc/driver/d3d12/d3d12_device.cpp

@@ -2579,6 +2579,9 @@ bool WrappedID3D12Device::ProcessChunk(ReadSerialiser &ser, D3D12Chunk context)
     case D3D12Chunk::Device_CreateHeapFromFileMapping:
       return Serialise_CreateHeap(ser, NULL, IID(), NULL);
       break;
+    case D3D12Chunk::Device_OpenSharedHandle:
+      return Serialise_OpenSharedHandle(ser, NULL, IID(), NULL);
+      break;
     default:
     {
       SystemChunk system = (SystemChunk)context;

renderdoc/driver/d3d12/d3d12_device_wrap.cpp

@@ -2132,10 +2132,180 @@ void WrappedID3D12Device::CopyDescriptorsSimple(UINT NumDescriptors,
   }
 }
 
+template <typename SerialiserType>
+bool WrappedID3D12Device::Serialise_OpenSharedHandle(SerialiserType &ser, HANDLE NTHandle,
+                                                     REFIID riid, void **ppvObj)
+{
+  SERIALISE_ELEMENT_LOCAL(ResourceRIID, riid);
+
+  SERIALISE_CHECK_READ_ERRORS();
+
+  bool isRes = (ResourceRIID == __uuidof(ID3D12Resource) ? true : false);
+  if(isRes)
+  {
+    D3D12_RESOURCE_DESC desc;
+    D3D12_HEAP_PROPERTIES heapProperties;
+    D3D12_HEAP_FLAGS heapFlags;
+
+    ID3D12Resource *res{};
+    if(ser.IsWriting())
+    {
+      res = (ID3D12Resource *)*ppvObj;
+      desc = res->GetDesc();
+      res->GetHeapProperties(&heapProperties, &heapFlags);
+    }
+
+    SERIALISE_ELEMENT_LOCAL(resourceId, GetResID(res));
+    SERIALISE_ELEMENT(desc);
+    SERIALISE_ELEMENT(heapProperties);
+    SERIALISE_ELEMENT(heapFlags);
+
+    if(IsReplayingAndReading())
+    {
+      D3D12_RESOURCE_STATES InitialResourceState = D3D12_RESOURCE_STATE_COMMON;
+      const D3D12_CLEAR_VALUE *pOptimizedClearValue = NULL;
+
+      // always allow SRVs on replay so we can inspect resources
+      desc.Flags &= ~D3D12_RESOURCE_FLAG_DENY_SHADER_RESOURCE;
+
+      HRESULT hr;
+      ID3D12Resource *ret;
+      hr = m_pDevice->CreateCommittedResource(&heapProperties, heapFlags, &desc, InitialResourceState,
+                                              pOptimizedClearValue, ResourceRIID, (void **)&ret);
+      if(FAILED(hr))
+      {
+        RDCERR("Failed on resource serialise-creation, HRESULT: %s", ToStr(hr).c_str());
+        return false;
+      }
+      else
+      {
+        SetObjName(ret, StringFormat::Fmt("Shared Resource %s ID %llu",
+                                          ToStr(desc.Dimension).c_str(), resourceId));
+
+        ret = new WrappedID3D12Resource(ret, this);
+
+        GetResourceManager()->AddLiveResource(resourceId, ret);
+
+        SubresourceStateVector &states = m_ResourceStates[GetResID(ret)];
+        states.resize(GetNumSubresources(m_pDevice, &desc), InitialResourceState);
+
+        ResourceType type = ResourceType::Texture;
+        const char *prefix = "Texture";
+
+        if(desc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER)
+        {
+          type = ResourceType::Buffer;
+          prefix = "Buffer";
+        }
+        else if(desc.Dimension == D3D12_RESOURCE_DIMENSION_TEXTURE1D)
+        {
+          prefix = desc.DepthOrArraySize > 1 ? "1D TextureArray" : "1D Texture";
+
+          if(desc.Flags & D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET)
+            prefix = "1D Render Target";
+          else if(desc.Flags & D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL)
+            prefix = "1D Depth Target";
+        }
+        else if(desc.Dimension == D3D12_RESOURCE_DIMENSION_TEXTURE2D)
+        {
+          prefix = desc.DepthOrArraySize > 1 ? "2D TextureArray" : "2D Texture";
+
+          if(desc.Flags & D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET)
+            prefix = "2D Render Target";
+          else if(desc.Flags & D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL)
+            prefix = "2D Depth Target";
+        }
+        else if(desc.Dimension == D3D12_RESOURCE_DIMENSION_TEXTURE3D)
+        {
+          prefix = "3D Texture";
+
+          if(desc.Flags & D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET)
+            prefix = "3D Render Target";
+          else if(desc.Flags & D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL)
+            prefix = "3D Depth Target";
+        }
+
+        AddResource(resourceId, type, prefix);
+      }
+    }
+  }
+  else
+  {
+    RDCERR("Unknown type of resource being shared");
+  }
+
+  return true;
+}
+
 HRESULT WrappedID3D12Device::OpenSharedHandle(HANDLE NTHandle, REFIID riid, void **ppvObj)
 {
-  D3D12NOTIMP("Shared Handles / API interop");
-  return m_pDevice->OpenSharedHandle(NTHandle, riid, ppvObj);
+  if(IsReplayMode(m_State))
+  {
+    RDCERR("Don't support opening shared handle during replay.");
+    return E_NOTIMPL;
+  }
+
+  if(ppvObj == NULL)
+    return E_INVALIDARG;
+
+  bool isRes = (riid == __uuidof(ID3D12Resource) ? true : false);
+  if(isRes)
+  {
+    void *ret = NULL;
+    HRESULT hr;
+    SERIALISE_TIME_CALL(hr = m_pDevice->OpenSharedHandle(NTHandle, riid, &ret));
+
+    if(FAILED(hr))
+    {
+      IUnknown *unk = (IUnknown *)ret;
+      SAFE_RELEASE(unk);
+      return hr;
+    }
+    else
+    {
+      ID3D12Resource *real = (ID3D12Resource *)ret;
+
+      WrappedID3D12Resource *wrapped = new WrappedID3D12Resource(real, this);
+      *ppvObj = (ID3D12Resource *)wrapped;
+
+      CACHE_THREAD_SERIALISER();
+
+      SCOPED_SERIALISE_CHUNK(D3D12Chunk::Device_OpenSharedHandle);
+      Serialise_OpenSharedHandle(ser, NTHandle, riid, ppvObj);
+
+      D3D12_RESOURCE_DESC desc = wrapped->GetDesc();
+      D3D12_RESOURCE_STATES InitialResourceState = D3D12_RESOURCE_STATE_COMMON;
+
+      D3D12ResourceRecord *record = GetResourceManager()->AddResourceRecord(wrapped->GetResourceID());
+      record->type = Resource_Resource;
+      record->Length = 0;
+      wrapped->SetResourceRecord(record);
+
+      record->m_MapsCount = GetNumSubresources(this, &desc);
+      record->m_Maps = new D3D12ResourceRecord::MapData[record->m_MapsCount];
+
+      record->AddChunk(scope.Get());
+
+      {
+        SCOPED_READLOCK(m_CapTransitionLock);
+        if(IsBackgroundCapturing(m_State))
+          GetResourceManager()->MarkDirtyResource(wrapped->GetResourceID());
+        else
+          GetResourceManager()->MarkPendingDirty(wrapped->GetResourceID());
+      }
+
+      {
+        SCOPED_LOCK(m_ResourceStatesLock);
+        SubresourceStateVector &states = m_ResourceStates[wrapped->GetResourceID()];
+
+        states.resize(GetNumSubresources(m_pDevice, &desc), InitialResourceState);
+      }
+
+      return S_OK;
+    }
+  }
+
+  return E_NOINTERFACE;
 }
 
 HRESULT WrappedID3D12Device::OpenSharedHandleByName(LPCWSTR Name, DWORD Access, HANDLE *pNTHandle)
@@ -2788,3 +2958,5 @@ INSTANTIATE_FUNCTION_SERIALISED(HRESULT, WrappedID3D12Device, EnqueueMakeResiden
                                 D3D12_RESIDENCY_FLAGS Flags, UINT NumObjects,
                                 ID3D12Pageable *const *ppObjects, ID3D12Fence *pFenceToSignal,
                                 UINT64 FenceValueToSignal);
+INSTANTIATE_FUNCTION_SERIALISED(void, WrappedID3D12Device, OpenSharedHandle, HANDLE NTHandle,
+                                REFIID riid, void **ppvObj);

renderdoc/driver/d3d12/d3d12_stringise.cpp

@@ -28,7 +28,9 @@
 template <>
 std::string DoStringise(const D3D12Chunk &el)
 {
-  RDCCOMPILE_ASSERT((uint32_t)D3D12Chunk::Max == 1096, "Chunks changed without updating names");
+  RDCCOMPILE_ASSERT((uint32_t)D3D12Chunk::Max == 1097, "Chunks changed without updating names");
+  RDCCOMPILE_ASSERT((uint32_t)D3D12Chunk::Device_OpenSharedHandle == 1096,
+                    "New chunks must be appended otherwise it breaks old captures");
 
   BEGIN_ENUM_STRINGISE(D3D12Chunk)
   {
@@ -70,6 +72,7 @@ std::string DoStringise(const D3D12Chunk &el)
     STRINGISE_ENUM_CLASS_NAMED(Device_CreateSampler, "ID3D12Device::CreateSampler");
     STRINGISE_ENUM_CLASS_NAMED(Device_CopyDescriptors, "ID3D12Device::CopyDescriptors");
     STRINGISE_ENUM_CLASS_NAMED(Device_CopyDescriptorsSimple, "ID3D12Device::CopyDescriptorsSimple");
+    STRINGISE_ENUM_CLASS_NAMED(Device_OpenSharedHandle, "ID3D12Device::OpenSharedHandle");
     STRINGISE_ENUM_CLASS_NAMED(Queue_ExecuteCommandLists,
                                "ID3D12CommandQueue::ExecuteCommandLists");
     STRINGISE_ENUM_CLASS_NAMED(Queue_Signal, "ID3D12CommandQueue::Signal");