Add support for AS copies on D3D12

* Most of these - all but a clone - require us to wait for the GPU execution to
  finish to get the size of the resulting AS to update our tracking. We issue a
  fence from the queue at submission time and then leave the RT manager to check
  for updates on each submission to get the latest information.

renderdoc/api/replay/replay_enums.h

@@ -5026,7 +5026,7 @@ actions.
 
 .. data:: BuildAccStruct
 
-  This action builds the acceleration structure.
+  This action builds or copies to and implicitly fills an acceleration structure.
 
 .. data:: Indexed
 

renderdoc/driver/d3d12/d3d12_command_list.h

@@ -273,7 +273,7 @@ public:
 
   void TakeWaitingASBuildCallbacks(rdcarray<std::function<bool()>> &callbacks)
   {
-    callbacks.append(m_PendingASCallbacks);
+    callbacks.append(std::move(m_PendingASCallbacks));
     m_PendingASCallbacks.clear();
   }
 

renderdoc/driver/d3d12/d3d12_command_list4_wrap.cpp

@@ -1100,10 +1100,72 @@ bool WrappedID3D12GraphicsCommandList::Serialise_EmitRaytracingAccelerationStruc
     _In_reads_(NumSourceAccelerationStructures)
         const D3D12_GPU_VIRTUAL_ADDRESS *pSourceAccelerationStructureData)
 {
-  // TODO AMD
-  RDCERR(
-      "EmitRaytracingAccelerationStructurePostbuildInfo called but raytracing is not supported!");
-  return false;
+  ID3D12GraphicsCommandList4 *pCommandList = this;
+  SERIALISE_ELEMENT(pCommandList);
+  SERIALISE_ELEMENT_LOCAL(Desc, *pDesc).Named("pDesc");
+  SERIALISE_ELEMENT(NumSourceAccelerationStructures).Important();
+  SERIALISE_ELEMENT_ARRAY_TYPED(D3D12BufferLocation, pSourceAccelerationStructureData,
+                                NumSourceAccelerationStructures);
+
+  SERIALISE_CHECK_READ_ERRORS();
+
+  if(IsReplayingAndReading())
+  {
+    if(GetWrapped(pCommandList)->GetReal4() == NULL)
+    {
+      SET_ERROR_RESULT(m_Cmd->m_FailedReplayResult, ResultCode::APIHardwareUnsupported,
+                       "Capture requires ID3D12GraphicsCommandList4 which isn't available");
+      return false;
+    }
+
+    if(m_pDevice->GetOpts5().RaytracingTier == D3D12_RAYTRACING_TIER_NOT_SUPPORTED)
+    {
+      SET_ERROR_RESULT(m_Cmd->m_FailedReplayResult, ResultCode::APIHardwareUnsupported,
+                       "Capture requires ray tracing support which isn't available");
+      return false;
+    }
+
+    m_Cmd->m_LastCmdListID = GetResourceManager()->GetOriginalID(GetResID(pCommandList));
+
+    if(IsActiveReplaying(m_State))
+    {
+      if(m_Cmd->InRerecordRange(m_Cmd->m_LastCmdListID))
+      {
+        Unwrap4(m_Cmd->RerecordCmdList(m_Cmd->m_LastCmdListID))
+            ->EmitRaytracingAccelerationStructurePostbuildInfo(
+                pDesc, NumSourceAccelerationStructures, pSourceAccelerationStructureData);
+      }
+    }
+    else
+    {
+      Unwrap4(pCommandList)
+          ->EmitRaytracingAccelerationStructurePostbuildInfo(pDesc, NumSourceAccelerationStructures,
+                                                             pSourceAccelerationStructureData);
+
+      m_Cmd->AddEvent();
+
+      ActionDescription action;
+      action.copyDestination = GetResourceManager()->GetOriginalID(
+          WrappedID3D12Resource::GetResIDFromAddr(pDesc->DestBuffer));
+      action.copyDestinationSubresource = Subresource();
+
+      action.flags |= ActionFlags::Copy;
+
+      m_Cmd->AddAction(action);
+
+      D3D12ActionTreeNode &actionNode = m_Cmd->GetActionStack().back()->children.back();
+
+      actionNode.resourceUsage.push_back(
+          make_rdcpair(WrappedID3D12Resource::GetResIDFromAddr(pDesc->DestBuffer),
+                       EventUsage(actionNode.action.eventId, ResourceUsage::CopyDst)));
+      for(UINT i = 0; i < NumSourceAccelerationStructures; i++)
+        actionNode.resourceUsage.push_back(make_rdcpair(
+            WrappedID3D12Resource::GetResIDFromAddr(pSourceAccelerationStructureData[i]),
+            EventUsage(actionNode.action.eventId, ResourceUsage::CopySrc)));
+    }
+  }
+
+  return true;
 }
 
 void WrappedID3D12GraphicsCommandList::EmitRaytracingAccelerationStructurePostbuildInfo(
@@ -1112,9 +1174,24 @@ void WrappedID3D12GraphicsCommandList::EmitRaytracingAccelerationStructurePostbu
     _In_reads_(NumSourceAccelerationStructures)
         const D3D12_GPU_VIRTUAL_ADDRESS *pSourceAccelerationStructureData)
 {
-  // TODO AMD
-  RDCERR(
-      "EmitRaytracingAccelerationStructurePostbuildInfo called but raytracing is not supported!");
+  SERIALISE_TIME_CALL(m_pList4->EmitRaytracingAccelerationStructurePostbuildInfo(
+      pDesc, NumSourceAccelerationStructures, pSourceAccelerationStructureData));
+
+  if(IsCaptureMode(m_State))
+  {
+    CACHE_THREAD_SERIALISER();
+    SCOPED_SERIALISE_CHUNK(D3D12Chunk::List_EmitRaytracingAccelerationStructurePostbuildInfo);
+    Serialise_EmitRaytracingAccelerationStructurePostbuildInfo(
+        ser, pDesc, NumSourceAccelerationStructures, pSourceAccelerationStructureData);
+
+    m_ListRecord->AddChunk(scope.Get(m_ListRecord->cmdInfo->alloc));
+    m_ListRecord->MarkResourceFrameReferenced(
+        WrappedID3D12Resource::GetResIDFromAddr(pDesc->DestBuffer), eFrameRef_PartialWrite);
+    for(UINT i = 0; i < NumSourceAccelerationStructures; i++)
+      m_ListRecord->MarkResourceFrameReferenced(
+          WrappedID3D12Resource::GetResIDFromAddr(pSourceAccelerationStructureData[i]),
+          eFrameRef_Read);
+  }
 }
 
 template <typename SerialiserType>
@@ -1123,8 +1200,45 @@ bool WrappedID3D12GraphicsCommandList::Serialise_CopyRaytracingAccelerationStruc
     _In_ D3D12_GPU_VIRTUAL_ADDRESS SourceAccelerationStructureData,
     _In_ D3D12_RAYTRACING_ACCELERATION_STRUCTURE_COPY_MODE Mode)
 {
-  // TODO AMD
-  return false;
+  ID3D12GraphicsCommandList4 *pCommandList = this;
+  SERIALISE_ELEMENT(pCommandList);
+  SERIALISE_ELEMENT_TYPED(D3D12BufferLocation, DestAccelerationStructureData)
+      .TypedAs("D3D12_GPU_VIRTUAL_ADDRESS"_lit)
+      .Important();
+  SERIALISE_ELEMENT_TYPED(D3D12BufferLocation, SourceAccelerationStructureData)
+      .TypedAs("D3D12_GPU_VIRTUAL_ADDRESS"_lit)
+      .Important();
+  SERIALISE_ELEMENT(Mode);
+
+  ID3D12GraphicsCommandList4 *dxrCmd = Unwrap4(pCommandList);
+
+  if(IsReplayingAndReading())
+  {
+    m_Cmd->m_LastCmdListID = GetResourceManager()->GetOriginalID(GetResID(pCommandList));
+
+    if(IsActiveReplaying(m_State))
+    {
+      if(m_Cmd->InRerecordRange(m_Cmd->m_LastCmdListID))
+      {
+        dxrCmd->CopyRaytracingAccelerationStructure(DestAccelerationStructureData,
+                                                    SourceAccelerationStructureData, Mode);
+      }
+    }
+    else
+    {
+      dxrCmd->CopyRaytracingAccelerationStructure(DestAccelerationStructureData,
+                                                  SourceAccelerationStructureData, Mode);
+
+      m_Cmd->AddEvent();
+
+      ActionDescription actionDesc;
+      actionDesc.flags |= ActionFlags::BuildAccStruct;
+      m_Cmd->AddAction(actionDesc);
+    }
+  }
+
+  SERIALISE_CHECK_READ_ERRORS();
+  return true;
 }
 
 void WrappedID3D12GraphicsCommandList::CopyRaytracingAccelerationStructure(
@@ -1132,8 +1246,115 @@ void WrappedID3D12GraphicsCommandList::CopyRaytracingAccelerationStructure(
     _In_ D3D12_GPU_VIRTUAL_ADDRESS SourceAccelerationStructureData,
     _In_ D3D12_RAYTRACING_ACCELERATION_STRUCTURE_COPY_MODE Mode)
 {
-  // TODO AMD
-  RDCERR("CopyRaytracingAccelerationStructure called but raytracing is not supported!");
+  SERIALISE_TIME_CALL(m_pList4->CopyRaytracingAccelerationStructure(
+      DestAccelerationStructureData, SourceAccelerationStructureData, Mode));
+
+  if(IsCaptureMode(m_State))
+  {
+    // Acceleration structure (AS) are created on buffer created with Acceleration structure init
+    // state which helps them differentiate between non-Acceleration structure buffers (non-ASB).
+
+    // AS creation at recording can happen at any offset, given offset + its size is less than the
+    // resource size. It can also be recorded for overwriting on same or another command list,
+    // invalidating occupying previous acceleration structure(s) in order of command list execution.
+    // It can also be updated but there are many update constraints around it.
+
+    {
+      CACHE_THREAD_SERIALISER();
+      SCOPED_SERIALISE_CHUNK(D3D12Chunk::List_CopyRaytracingAccelerationStructure);
+      Serialise_CopyRaytracingAccelerationStructure(ser, DestAccelerationStructureData,
+                                                    SourceAccelerationStructureData, Mode);
+
+      m_ListRecord->AddChunk(scope.Get(m_ListRecord->cmdInfo->alloc));
+    }
+
+    ResourceId destASBId;
+    D3D12BufferOffset destASBOffset;
+
+    WrappedID3D12Resource::GetResIDFromAddr(DestAccelerationStructureData, destASBId, destASBOffset);
+
+    if(Mode != D3D12_RAYTRACING_ACCELERATION_STRUCTURE_COPY_MODE_SERIALIZE &&
+       Mode != D3D12_RAYTRACING_ACCELERATION_STRUCTURE_COPY_MODE_VISUALIZATION_DECODE_FOR_TOOLS)
+    {
+      // these outputs are not ASs themselves, so we don't need to do any further tracking
+    }
+    else if(Mode == D3D12_RAYTRACING_ACCELERATION_STRUCTURE_COPY_MODE_DESERIALIZE ||
+            Mode == D3D12_RAYTRACING_ACCELERATION_STRUCTURE_COPY_MODE_COMPACT)
+    {
+      // this needs special handling because the size of the destination AS is not known in advance
+      // on the CPU.
+      // For serialisation the size is given by the serialisation data - it's stored in
+      // the header of the serialisation data.
+      // For compaction the size is determined by an EmitPostbuildInfo. We could do this earlier,
+      // when the AS is first built but that would still require an asynchronous read and it could
+      // be built in the same command buffer that it's compacted.
+      //
+      // In either case we want to late-read this size. We could grab the size directly from the
+      // serialised data but instead we just do a postbuild info of the current size of the dest
+      // after the copy operation for simplicity.
+
+      D3D12GpuBuffer *sizeBuffer = NULL;
+      D3D12GpuBufferAllocator::Inst()->Alloc(D3D12GpuBufferHeapType::CustomHeapWithUavCpuAccess,
+                                             D3D12GpuBufferHeapMemoryFlag::Default, 8, 8,
+                                             &sizeBuffer);
+
+      D3D12_RAYTRACING_ACCELERATION_STRUCTURE_POSTBUILD_INFO_DESC desc = {};
+      desc.DestBuffer = sizeBuffer->Address();
+      desc.InfoType = D3D12_RAYTRACING_ACCELERATION_STRUCTURE_POSTBUILD_INFO_CURRENT_SIZE;
+
+      D3D12_RESOURCE_BARRIER barrier = {};
+      barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_UAV;
+
+      // wait for build to finish
+      m_pList4->ResourceBarrier(1, &barrier);
+
+      // query current size
+      m_pList4->EmitRaytracingAccelerationStructurePostbuildInfo(&desc, 1,
+                                                                 &DestAccelerationStructureData);
+
+      auto PostBldExecute = [this, destASBId, destASBOffset, sizeBuffer]() -> bool {
+        UINT64 *size = (UINT64 *)sizeBuffer->Map();
+        UINT64 destSize = *size;
+        sizeBuffer->Unmap();
+        sizeBuffer->Release();
+
+        return ProcessASBuildAfterSubmission(destASBId, destASBOffset, destSize);
+      };
+
+      AddSubmissionASBuildCallback(true, PostBldExecute);
+    }
+    else
+    {
+      ResourceId srcASBId;
+      D3D12BufferOffset srcASBOffset;
+
+      WrappedID3D12Resource::GetResIDFromAddr(SourceAccelerationStructureData, srcASBId,
+                                              srcASBOffset);
+
+      // simple clone - easy case where size is known from the source
+      // this does _not_ strictly need to be marked as pending - we could process it immediately in
+      // almost every case. However if this copy's source comes from a copy itself that is not
+      // directly processed then we need to defer this to wait until the source acceleration
+      // structure is up to date. Deferring this should not cause a problem as we will still have it
+      // up to date before any subsequent work that depends on it like beginning a capture.
+      AddSubmissionASBuildCallback(true, [this, destASBId, destASBOffset, srcASBId, srcASBOffset]() {
+        D3D12ResourceManager *resManager = m_pDevice->GetResourceManager();
+
+        D3D12AccelerationStructure *accStructAtSrcOffset = NULL;
+
+        WrappedID3D12Resource *srcASB = resManager->GetCurrentAs<WrappedID3D12Resource>(srcASBId);
+
+        // get the source AS, we should have this and can't proceed without it to give us the size
+        if(!srcASB->GetAccStructIfExist(srcASBOffset, &accStructAtSrcOffset))
+        {
+          RDCERR("Couldn't find source acceleration structure in AS copy");
+          return false;
+        }
+
+        return ProcessASBuildAfterSubmission(destASBId, destASBOffset, accStructAtSrcOffset->Size());
+      });
+    }
+  }
 }
 
 template <typename SerialiserType>

renderdoc/driver/d3d12/d3d12_command_queue.h

@@ -176,6 +176,8 @@ class WrappedID3D12CommandQueue : public ID3D12CommandQueue,
   UINT64 m_RayFenceValue = 1;
   rdcarray<PatchedRayDispatch::Resources> m_RayDispatchesPending;
 
+  ID3D12Fence *GetRayFence();
+
   bool m_MarkedActive = false;
 
   WrappedID3D12DebugCommandQueue m_WrappedDebug;

renderdoc/driver/d3d12/d3d12_command_queue_wrap.cpp

@@ -709,6 +709,20 @@ bool WrappedID3D12CommandQueue::Serialise_ExecuteCommandLists(SerialiserType &se
   return true;
 }
 
+ID3D12Fence *WrappedID3D12CommandQueue::GetRayFence()
+{
+  // if we don't have a fence for this queue tracking, create it now
+  if(!m_RayFence)
+  {
+    // create this unwrapped so that it doesn't get recorded into captures
+    m_pDevice->GetReal()->CreateFence(0, D3D12_FENCE_FLAG_NONE, __uuidof(ID3D12Fence),
+                                      (void **)&m_RayFence);
+    m_RayFence->SetName(L"Queue Ray Fence");
+  }
+
+  return m_RayFence;
+}
+
 void WrappedID3D12CommandQueue::ExecuteCommandLists(UINT NumCommandLists,
                                                     ID3D12CommandList *const *ppCommandLists)
 {
@@ -739,6 +753,8 @@ void WrappedID3D12CommandQueue::ExecuteCommandListsInternal(UINT NumCommandLists
   {
     SERIALISE_TIME_CALL(m_pReal->ExecuteCommandLists(NumCommandLists, unwrapped));
 
+    rdcarray<std::function<bool()>> pendingASBuildCallbacks;
+
     for(UINT i = 0; i < NumCommandLists; i++)
     {
       WrappedID3D12GraphicsCommandList *wrapped =
@@ -748,7 +764,29 @@ void WrappedID3D12CommandQueue::ExecuteCommandListsInternal(UINT NumCommandLists
       {
         RDCERR("Unable to execute post build for acc struct");
       }
+
+      wrapped->TakeWaitingASBuildCallbacks(pendingASBuildCallbacks);
     }
+
+    if(!pendingASBuildCallbacks.empty())
+    {
+      ID3D12Fence *fence = GetRayFence();
+
+      // these callbacks need to be synchronised at every submission to process them as soon as the
+      // results are available, since we could submit a build on one queue and then a dependent
+      // build on another queue later once it's finished without any intermediate submissions on the
+      // first queue. For that reason we pass these to the RT handler to hold onto, and tick it
+      GetResourceManager()->GetRaytracingResourceAndUtilHandler()->AddPendingASBuilds(
+          fence, m_RayFenceValue, pendingASBuildCallbacks);
+
+      // add the signal for those callbacks to wait on
+      HRESULT hr = m_pReal->Signal(fence, m_RayFenceValue++);
+      m_pDevice->CheckHRESULT(hr);
+      RDCASSERTEQUAL(hr, S_OK);
+    }
+
+    // check AS builds now
+    GetResourceManager()->GetRaytracingResourceAndUtilHandler()->CheckPendingASBuilds();
   }
 
   if(IsCaptureMode(m_State))
@@ -890,21 +928,12 @@ void WrappedID3D12CommandQueue::ExecuteCommandListsInternal(UINT NumCommandLists
 
     if(!rayDispatches.empty())
     {
-      // if we don't have a fence for this queue tracking, create it now
-      if(!m_RayFence)
-      {
-        // create this unwrapped so that it doesn't get recorded into captures
-        m_pDevice->GetReal()->CreateFence(0, D3D12_FENCE_FLAG_NONE, __uuidof(ID3D12Fence),
-                                          (void **)&m_RayFence);
-        m_RayFence->SetName(L"Queue Ray Fence");
-      }
-
       for(PatchedRayDispatch::Resources &ray : rayDispatches)
         ray.fenceValue = m_RayFenceValue;
 
       m_RayDispatchesPending.append(rayDispatches);
 
-      HRESULT hr = m_pReal->Signal(m_RayFence, m_RayFenceValue++);
+      HRESULT hr = m_pReal->Signal(GetRayFence(), m_RayFenceValue++);
       m_pDevice->CheckHRESULT(hr);
       RDCASSERTEQUAL(hr, S_OK);
     }

renderdoc/driver/d3d12/d3d12_device.cpp

@@ -2591,6 +2591,9 @@ void WrappedID3D12Device::StartFrameCapture(DeviceOwnedWindow devWnd)
 
     GPUSyncAllQueues();
 
+    // wait until we've synced all queues to check for these
+    GetResourceManager()->GetRaytracingResourceAndUtilHandler()->CheckPendingASBuilds();
+
     GetResourceManager()->PrepareInitialContents();
 
     if(initStateCurList)

renderdoc/driver/d3d12/d3d12_manager.cpp

@@ -782,6 +782,43 @@ void D3D12RaytracingResourceAndUtilHandler::ResizeSerialisationBuffer(UINT64 siz
   }
 }
 
+void D3D12RaytracingResourceAndUtilHandler::AddPendingASBuilds(
+    ID3D12Fence *fence, UINT64 waitValue, const rdcarray<std::function<bool()>> &callbacks)
+{
+  SCOPED_LOCK(m_PendingASBuildsLock);
+  for(const std::function<bool()> &cb : callbacks)
+  {
+    fence->AddRef();
+    m_PendingASBuilds.push_back({fence, waitValue, cb});
+  }
+}
+
+void D3D12RaytracingResourceAndUtilHandler::CheckPendingASBuilds()
+{
+  std::map<ID3D12Fence *, UINT64> fenceValues;
+  SCOPED_LOCK(m_PendingASBuildsLock);
+
+  if(m_PendingASBuilds.empty())
+    return;
+
+  for(PendingASBuild &build : m_PendingASBuilds)
+  {
+    // first time we see each fence, get the completed value
+    if(fenceValues[build.fence] == 0)
+      fenceValues[build.fence] = build.fence->GetCompletedValue();
+
+    // if this fence has been satisfied, release our ref (so it will also get removed below) and call the callback
+    if(fenceValues[build.fence] >= build.fenceValue)
+    {
+      SAFE_RELEASE(build.fence);
+      build.callback();
+    }
+  }
+
+  // remove any builds that completed
+  m_PendingASBuilds.removeIf([](const PendingASBuild &build) { return build.fence == NULL; });
+}
+
 PatchedRayDispatch D3D12RaytracingResourceAndUtilHandler::PatchRayDispatch(
     ID3D12GraphicsCommandList4 *unwrappedCmd, rdcarray<ResourceId> heaps,
     const D3D12_DISPATCH_RAYS_DESC &desc)

renderdoc/driver/d3d12/d3d12_manager.h

@@ -1113,6 +1113,10 @@ public:
                                       rdcarray<ResourceId> heaps,
                                       const D3D12_DISPATCH_RAYS_DESC &desc);
 
+  void AddPendingASBuilds(ID3D12Fence *fence, UINT64 waitValue,
+                          const rdcarray<std::function<bool()>> &callbacks);
+  void CheckPendingASBuilds();
+
   void ResizeSerialisationBuffer(UINT64 size);
 
   // buffer in UAV state for emitting AS queries to, CPU accessible/mappable
@@ -1157,6 +1161,15 @@ private:
     ID3D12RootSignature *rootSig = NULL;
     ID3D12PipelineState *pipe = NULL;
   } m_RayPatchingData;
+
+  struct PendingASBuild
+  {
+    ID3D12Fence *fence;
+    UINT64 fenceValue;
+    std::function<bool()> callback;
+  };
+  Threading::CriticalSection m_PendingASBuildsLock;
+  rdcarray<PendingASBuild> m_PendingASBuilds;
 };
 
 struct D3D12ResourceManagerConfiguration

renderdoc/serialise/serialiser.h

@@ -2101,6 +2101,18 @@ struct ScopedDeserialiseArray<SerialiserType, byte *>
   CONCAT(union, __LINE__).o = &obj;        \
   GET_SERIALISER.template Serialise<type>(STRING_LITERAL(#obj), *CONCAT(union, __LINE__).t)
 
+#define SERIALISE_ELEMENT_ARRAY_TYPED(type, arrayObj, countObj)                                  \
+  RDCCOMPILE_ASSERT(sizeof(*arrayObj) == sizeof(type),                                           \
+                    "Array serialised co-erced type must be identically sized");                 \
+  union                                                                                          \
+  {                                                                                              \
+    type **t;                                                                                    \
+    decltype(arrayObj) *o;                                                                       \
+  } CONCAT(union, __LINE__);                                                                     \
+  CONCAT(union, __LINE__).o = &arrayObj;                                                         \
+  GET_SERIALISER.template Serialise<type>(STRING_LITERAL(#arrayObj), *CONCAT(union, __LINE__).t, \
+                                          countObj, SerialiserFlags::AllocateMemory)
+
 #define SERIALISE_ELEMENT_ARRAY(obj, count)                                                       \
   uint64_t CONCAT(dummy_array_count, __LINE__) = 0;                                               \
   (void)CONCAT(dummy_array_count, __LINE__);                                                      \