Don't allow command pool/allocator reset until capture is finished

* Even after transitioning to background capturing we can't allow reset or else
  we could record over chunks that we have yet to write to disk.

renderdoc/driver/d3d12/d3d12_debug.cpp

@@ -345,7 +345,6 @@ D3D12DebugManager::D3D12DebugManager(WrappedID3D12Device *wrapper)
 
   hr = m_pDevice->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT,
                                          __uuidof(ID3D12CommandAllocator), (void **)&m_DebugAlloc);
-  ((WrappedID3D12CommandAllocator *)m_DebugAlloc)->SetInternal(true);
   m_pDevice->InternalRef();
 
   if(FAILED(hr))

renderdoc/driver/d3d12/d3d12_device.cpp

@@ -2136,6 +2136,8 @@ void WrappedID3D12Device::StartFrameCapture(void *dev, void *wnd)
 
   RDCLOG("Starting capture");
 
+  WrappedID3D12CommandAllocator::PauseResets();
+
   m_CaptureTimer.Restart();
 
   m_AppControlledCapture = true;
@@ -2531,6 +2533,8 @@ bool WrappedID3D12Device::EndFrameCapture(void *dev, void *wnd)
   for(ID3D12Resource *r : m_RefBuffers)
     r->Release();
 
+  WrappedID3D12CommandAllocator::ResumeResets();
+
   GetResourceManager()->MarkUnwrittenResources();
 
   GetResourceManager()->ClearReferencedResources();
@@ -2583,6 +2587,8 @@ bool WrappedID3D12Device::DiscardFrameCapture(void *dev, void *wnd)
   for(ID3D12Resource *r : m_RefBuffers)
     r->Release();
 
+  WrappedID3D12CommandAllocator::ResumeResets();
+
   GetResourceManager()->MarkUnwrittenResources();
 
   GetResourceManager()->ClearReferencedResources();
@@ -3190,7 +3196,6 @@ void WrappedID3D12Device::CreateInternalResources()
 
   CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, __uuidof(ID3D12CommandAllocator),
                          (void **)&m_Alloc);
-  ((WrappedID3D12CommandAllocator *)m_Alloc)->SetInternal(true);
   InternalRef();
   CreateFence(0, D3D12_FENCE_FLAG_NONE, __uuidof(ID3D12Fence), (void **)&m_GPUSyncFence);
   InternalRef();
@@ -3201,7 +3206,6 @@ void WrappedID3D12Device::CreateInternalResources()
 
   CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, __uuidof(ID3D12CommandAllocator),
                          (void **)&m_DataUploadAlloc);
-  ((WrappedID3D12CommandAllocator *)m_DataUploadAlloc)->SetInternal(true);
   InternalRef();
 
   GetResourceManager()->SetInternalResource(m_DataUploadAlloc);

renderdoc/driver/d3d12/d3d12_resources.cpp

@@ -30,6 +30,7 @@
 GPUAddressRangeTracker WrappedID3D12Resource::m_Addresses;
 std::map<WrappedID3D12PipelineState::DXBCKey, WrappedID3D12Shader *> WrappedID3D12Shader::m_Shaders;
 bool WrappedID3D12Shader::m_InternalResources = false;
+int32_t WrappedID3D12CommandAllocator::m_ResetEnabled = 1;
 
 const GUID RENDERDOC_ID3D12ShaderGUID_ShaderDebugMagicValue = RENDERDOC_ShaderDebugMagicValue_struct;
 

renderdoc/driver/d3d12/d3d12_resources.h

@@ -328,6 +328,8 @@ public:
 
 class WrappedID3D12CommandAllocator : public WrappedDeviceChild12<ID3D12CommandAllocator>
 {
+  static int32_t m_ResetEnabled;
+
 public:
   ALLOCATE_WITH_WRAPPED_POOL(WrappedID3D12CommandAllocator);
 
@@ -344,18 +346,15 @@ public:
   {
   }
   virtual ~WrappedID3D12CommandAllocator() { Shutdown(); }
-  void SetInternal(bool internalAlloc) { m_Internal = internalAlloc; }
+  static void PauseResets() { Atomic::Dec32(&m_ResetEnabled); }
+  static void ResumeResets() { Atomic::Inc32(&m_ResetEnabled); }
   //////////////////////////////
   // implement ID3D12CommandAllocator
 
   virtual HRESULT STDMETHODCALLTYPE Reset()
   {
-    if(!m_Internal)
-      m_pDevice->GetCapTransitionLock().ReadLock();
-    if(IsBackgroundCapturing(m_pDevice->GetState()))
+    if(Atomic::CmpExch32(&m_ResetEnabled, 1, 1) == 1)
       alloc.Reset();
-    if(!m_Internal)
-      m_pDevice->GetCapTransitionLock().ReadUnlock();
     return m_pReal->Reset();
   }
 };

renderdoc/driver/vulkan/wrappers/vk_cmd_funcs.cpp

@@ -546,11 +546,8 @@ VkResult WrappedVulkan::vkCreateCommandPool(VkDevice device,
 VkResult WrappedVulkan::vkResetCommandPool(VkDevice device, VkCommandPool cmdPool,
                                            VkCommandPoolResetFlags flags)
 {
-  {
-    SCOPED_READLOCK(m_CapTransitionLock);
-    if(IsBackgroundCapturing(m_State))
-      GetRecord(cmdPool)->cmdPoolInfo->alloc.Reset();
-  }
+  if(Atomic::CmpExch32(&m_ReuseEnabled, 1, 1) == 1)
+    GetRecord(cmdPool)->cmdPoolInfo->alloc.Reset();
 
   return ObjDisp(device)->ResetCommandPool(Unwrap(device), Unwrap(cmdPool), flags);
 }