When forcing all resources in for bindless on D3D12, be more granular

* We only need to force on actual D3D12 resources (buffers and textures) -
  enabling the Ref All Resources also would include all pipelines and other
  objects which do not need to be included and could be quite wasteful.

renderdoc/driver/d3d12/d3d12_command_list_wrap.cpp

@@ -1767,7 +1767,7 @@ void WrappedID3D12GraphicsCommandList::SetComputeRootDescriptorTable(
         num = HeapNumDescriptors - rangeStart->GetHeapIndex();
       }
 
-      if(!RenderDoc::Inst().GetCaptureOptions().refAllResources)
+      if(!m_pDevice->IsBindlessResourceUseActive())
       {
         rdcarray<rdcpair<D3D12Descriptor *, UINT>> &descs = m_ListRecord->cmdInfo->boundDescs;
         descs.push_back(make_rdcpair(rangeStart, num));
@@ -2344,7 +2344,7 @@ void WrappedID3D12GraphicsCommandList::SetGraphicsRootDescriptorTable(
         num = HeapNumDescriptors - rangeStart->GetHeapIndex();
       }
 
-      if(!RenderDoc::Inst().GetCaptureOptions().refAllResources)
+      if(!m_pDevice->IsBindlessResourceUseActive())
       {
         rdcarray<rdcpair<D3D12Descriptor *, UINT>> &descs = m_ListRecord->cmdInfo->boundDescs;
         descs.push_back(make_rdcpair(rangeStart, num));

renderdoc/driver/d3d12/d3d12_device.cpp

@@ -2362,6 +2362,14 @@ void WrappedID3D12Device::StartFrameCapture(DeviceOwnedWindow devWnd)
                                                         eFrameRef_Read);
     }
 
+    if(m_BindlessResourceUseActive)
+    {
+      SCOPED_LOCK(m_ResourceStatesLock);
+
+      for(auto it = m_BindlessFrameRefs.begin(); it != m_BindlessFrameRefs.end(); ++it)
+        GetResourceManager()->MarkResourceFrameReferenced(it->first, it->second);
+    }
+
     m_RefQueues = m_Queues;
     m_RefBuffers = WrappedID3D12Resource::AddRefBuffersBeforeCapture(GetResourceManager());
   }
@@ -3333,6 +3341,18 @@ HRESULT STDMETHODCALLTYPE WrappedID3D12Device::QueryVideoMemoryInfo(
   return m_pDownlevel->QueryVideoMemoryInfo(NodeIndex, MemorySegmentGroup, pVideoMemoryInfo);
 }
 
+FrameRefType WrappedID3D12Device::BindlessRefTypeForRes(ID3D12Resource *wrapped)
+{
+  // if the resource could be used in any mutable way, assume it's written. Otherwise assume it's
+  // read-only (if it's modified some other way like with a copy etc, this will naturally become a
+  // read-before-write)
+  return (wrapped->GetDesc().Flags &
+          (D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET | D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL |
+           D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS)) != 0
+             ? eFrameRef_ReadBeforeWrite
+             : eFrameRef_Read;
+}
+
 byte *WrappedID3D12Device::GetTempMemory(size_t s)
 {
   TempMem *mem = (TempMem *)Threading::GetTLSValue(tempMemoryTLSSlot);

renderdoc/driver/d3d12/d3d12_device.h

@@ -717,6 +717,7 @@ private:
   // used both on capture and replay side to track resource states. Only locked
   // in capture
   std::map<ResourceId, SubresourceStateVector> m_ResourceStates;
+  std::unordered_map<ResourceId, FrameRefType> m_BindlessFrameRefs;
   Threading::CriticalSection m_ResourceStatesLock;
 
   // used on replay only. Contains the initial resource states before any barriers - this allows us
@@ -737,6 +738,9 @@ private:
     WrappedID3D12CommandQueue *queue;
   };
 
+  bool m_BindlessResourceUseActive = false;
+  FrameRefType BindlessRefTypeForRes(ID3D12Resource *wrapped);
+
   std::map<IDXGISwapper *, SwapPresentInfo> m_SwapChains;
   std::map<ResourceId, DXGI_FORMAT> m_BackbufferFormat;
 
@@ -801,6 +805,7 @@ public:
   void FirstFrame(IDXGISwapper *swapper);
   const ActionDescription *GetAction(uint32_t eventId);
 
+  bool IsBindlessResourceUseActive() const { return m_BindlessResourceUseActive; }
   ResourceId GetFrameCaptureResourceId() { return m_FrameCaptureRecord->GetResourceID(); }
   void AddDebugMessage(MessageCategory c, MessageSeverity sv, MessageSource src, rdcstr d);
   void AddDebugMessage(const DebugMessage &msg);

renderdoc/driver/d3d12/d3d12_device_wrap.cpp

@@ -1102,48 +1102,42 @@ HRESULT WrappedID3D12Device::CreateRootSignature(UINT nodeMask, const void *pBlo
 
       wrapped->sig = GetShaderCache()->GetRootSig(pBlobWithRootSignature, blobLengthInBytes);
 
-      bool forceRefAll = false;
-
-      // force ref-all-resources if the heap is directly indexed because we can't track resource
-      // access
-      if(wrapped->sig.Flags & (D3D12_ROOT_SIGNATURE_FLAG_CBV_SRV_UAV_HEAP_DIRECTLY_INDEXED |
-                               D3D12_ROOT_SIGNATURE_FLAG_SAMPLER_HEAP_DIRECTLY_INDEXED))
+      if(!m_BindlessResourceUseActive)
       {
-        forceRefAll = true;
-        RDCDEBUG("Forcing Ref All Resources due to heap-indexing root signature flags");
-      }
-      else
-      {
-        for(const D3D12RootSignatureParameter &param : wrapped->sig.Parameters)
+        // force ref-all-resources if the heap is directly indexed because we can't track resource
+        // access
+        if(wrapped->sig.Flags & (D3D12_ROOT_SIGNATURE_FLAG_CBV_SRV_UAV_HEAP_DIRECTLY_INDEXED |
+                                 D3D12_ROOT_SIGNATURE_FLAG_SAMPLER_HEAP_DIRECTLY_INDEXED))
         {
-          if(param.ParameterType != D3D12_ROOT_PARAMETER_TYPE_DESCRIPTOR_TABLE)
-            continue;
-
-          for(UINT r = 0; r < param.DescriptorTable.NumDescriptorRanges; r++)
-          {
-            const D3D12_DESCRIPTOR_RANGE1 &range = param.DescriptorTable.pDescriptorRanges[r];
-            if(range.NumDescriptors > 100000)
-            {
-              forceRefAll = true;
-              RDCDEBUG(
-                  "Forcing Ref All Resources due to large root signature range of %u descriptors "
-                  "(space=%u, reg=%u, visibility=%s)",
-                  range.NumDescriptors, range.RegisterSpace, range.BaseShaderRegister,
-                  ToStr(param.ShaderVisibility).c_str());
-              break;
-            }
-          }
-
-          if(forceRefAll)
-            break;
+          m_BindlessResourceUseActive = true;
+          RDCDEBUG("Forcing Ref All Resources due to heap-indexing root signature flags");
         }
-      }
+        else
+        {
+          for(const D3D12RootSignatureParameter &param : wrapped->sig.Parameters)
+          {
+            if(param.ParameterType != D3D12_ROOT_PARAMETER_TYPE_DESCRIPTOR_TABLE)
+              continue;
 
-      if(forceRefAll)
-      {
-        CaptureOptions opts = RenderDoc::Inst().GetCaptureOptions();
-        opts.refAllResources = true;
-        RenderDoc::Inst().SetCaptureOptions(opts);
+            for(UINT r = 0; r < param.DescriptorTable.NumDescriptorRanges; r++)
+            {
+              const D3D12_DESCRIPTOR_RANGE1 &range = param.DescriptorTable.pDescriptorRanges[r];
+              if(range.NumDescriptors > 100000)
+              {
+                m_BindlessResourceUseActive = true;
+                RDCDEBUG(
+                    "Forcing Ref All Resources due to large root signature range of %u descriptors "
+                    "(space=%u, reg=%u, visibility=%s)",
+                    range.NumDescriptors, range.RegisterSpace, range.BaseShaderRegister,
+                    ToStr(param.ShaderVisibility).c_str());
+                break;
+              }
+            }
+
+            if(m_BindlessResourceUseActive)
+              break;
+          }
+        }
       }
 
       record->AddChunk(scope.Get());
@@ -1630,6 +1624,8 @@ HRESULT WrappedID3D12Device::CreateCommittedResource(const D3D12_HEAP_PROPERTIES
       SubresourceStateVector &states = m_ResourceStates[wrapped->GetResourceID()];
 
       states.fill(GetNumSubresources(m_pDevice, pDesc), InitialResourceState);
+
+      m_BindlessFrameRefs[wrapped->GetResourceID()] = BindlessRefTypeForRes(wrapped);
     }
 
     if(riidResource == __uuidof(ID3D12Resource))
@@ -1648,6 +1644,9 @@ HRESULT WrappedID3D12Device::CreateCommittedResource(const D3D12_HEAP_PROPERTIES
       {
         wrapped->AddRef();
         m_RefBuffers.push_back(wrapped);
+        if(m_BindlessResourceUseActive)
+          GetResourceManager()->MarkResourceFrameReferenced(wrapped->GetResourceID(),
+                                                            BindlessRefTypeForRes(wrapped));
       }
     }
   }
@@ -1945,6 +1944,8 @@ HRESULT WrappedID3D12Device::CreatePlacedResource(ID3D12Heap *pHeap, UINT64 Heap
       SubresourceStateVector &states = m_ResourceStates[wrapped->GetResourceID()];
 
       states.fill(GetNumSubresources(m_pDevice, pDesc), InitialState);
+
+      m_BindlessFrameRefs[wrapped->GetResourceID()] = BindlessRefTypeForRes(wrapped);
     }
 
     if(riid == __uuidof(ID3D12Resource))
@@ -1963,6 +1964,9 @@ HRESULT WrappedID3D12Device::CreatePlacedResource(ID3D12Heap *pHeap, UINT64 Heap
       {
         wrapped->AddRef();
         m_RefBuffers.push_back(wrapped);
+        if(m_BindlessResourceUseActive)
+          GetResourceManager()->MarkResourceFrameReferenced(wrapped->GetResourceID(),
+                                                            BindlessRefTypeForRes(wrapped));
       }
     }
   }
@@ -2183,6 +2187,8 @@ HRESULT WrappedID3D12Device::CreateReservedResource(const D3D12_RESOURCE_DESC *p
       SubresourceStateVector &states = m_ResourceStates[wrapped->GetResourceID()];
 
       states.fill(GetNumSubresources(m_pDevice, pDesc), InitialState);
+
+      m_BindlessFrameRefs[wrapped->GetResourceID()] = BindlessRefTypeForRes(wrapped);
     }
 
     if(riid == __uuidof(ID3D12Resource))
@@ -2201,6 +2207,9 @@ HRESULT WrappedID3D12Device::CreateReservedResource(const D3D12_RESOURCE_DESC *p
       {
         wrapped->AddRef();
         m_RefBuffers.push_back(wrapped);
+        if(m_BindlessResourceUseActive)
+          GetResourceManager()->MarkResourceFrameReferenced(wrapped->GetResourceID(),
+                                                            BindlessRefTypeForRes(wrapped));
       }
     }
   }
@@ -3132,6 +3141,8 @@ HRESULT WrappedID3D12Device::OpenSharedHandleInternal(D3D12Chunk chunkType,
         SubresourceStateVector &states = m_ResourceStates[wrapped->GetResourceID()];
 
         states.fill(GetNumSubresources(m_pDevice, &desc), InitialResourceState);
+
+        m_BindlessFrameRefs[wrapped->GetResourceID()] = BindlessRefTypeForRes(wrapped);
       }
 
       // while actively capturing we keep all buffers around to prevent the address lookup from
@@ -3143,6 +3154,9 @@ HRESULT WrappedID3D12Device::OpenSharedHandleInternal(D3D12Chunk chunkType,
         {
           wrapped->AddRef();
           m_RefBuffers.push_back(wrapped);
+          if(m_BindlessResourceUseActive)
+            GetResourceManager()->MarkResourceFrameReferenced(wrapped->GetResourceID(),
+                                                              BindlessRefTypeForRes(wrapped));
         }
       }
     }

renderdoc/driver/d3d12/d3d12_device_wrap4.cpp

@@ -422,6 +422,8 @@ HRESULT WrappedID3D12Device::CreateCommittedResource1(
       SubresourceStateVector &states = m_ResourceStates[wrapped->GetResourceID()];
 
       states.fill(GetNumSubresources(m_pDevice, pDesc), InitialResourceState);
+
+      m_BindlessFrameRefs[wrapped->GetResourceID()] = BindlessRefTypeForRes(wrapped);
     }
 
     if(riidResource == __uuidof(ID3D12Resource))
@@ -440,6 +442,9 @@ HRESULT WrappedID3D12Device::CreateCommittedResource1(
       {
         wrapped->AddRef();
         m_RefBuffers.push_back(wrapped);
+        if(m_BindlessResourceUseActive)
+          GetResourceManager()->MarkResourceFrameReferenced(wrapped->GetResourceID(),
+                                                            BindlessRefTypeForRes(wrapped));
       }
     }
   }

renderdoc/driver/d3d12/d3d12_device_wrap8.cpp

@@ -252,6 +252,8 @@ HRESULT WrappedID3D12Device::CreateCommittedResource2(
       D3D12_RESOURCE_DESC desc0;
       memcpy(&desc0, pDesc, sizeof(desc0));
       states.fill(GetNumSubresources(m_pDevice, &desc0), InitialResourceState);
+
+      m_BindlessFrameRefs[wrapped->GetResourceID()] = BindlessRefTypeForRes(wrapped);
     }
 
     if(riidResource == __uuidof(ID3D12Resource))
@@ -270,6 +272,9 @@ HRESULT WrappedID3D12Device::CreateCommittedResource2(
       {
         wrapped->AddRef();
         m_RefBuffers.push_back(wrapped);
+        if(m_BindlessResourceUseActive)
+          GetResourceManager()->MarkResourceFrameReferenced(wrapped->GetResourceID(),
+                                                            BindlessRefTypeForRes(wrapped));
       }
     }
   }
@@ -486,6 +491,8 @@ HRESULT WrappedID3D12Device::CreatePlacedResource1(ID3D12Heap *pHeap, UINT64 Hea
       D3D12_RESOURCE_DESC desc0;
       memcpy(&desc0, pDesc, sizeof(desc0));
       states.fill(GetNumSubresources(m_pDevice, &desc0), InitialState);
+
+      m_BindlessFrameRefs[wrapped->GetResourceID()] = BindlessRefTypeForRes(wrapped);
     }
 
     if(riid == __uuidof(ID3D12Resource))
@@ -504,6 +511,9 @@ HRESULT WrappedID3D12Device::CreatePlacedResource1(ID3D12Heap *pHeap, UINT64 Hea
       {
         wrapped->AddRef();
         m_RefBuffers.push_back(wrapped);
+        if(m_BindlessResourceUseActive)
+          GetResourceManager()->MarkResourceFrameReferenced(wrapped->GetResourceID(),
+                                                            BindlessRefTypeForRes(wrapped));
       }
     }
   }