Make internal command list etc wrapped, support multiple record lists

renderdoc/driver/d3d12/d3d12_command_queue_wrap.cpp

@@ -87,83 +87,86 @@ void STDMETHODCALLTYPE WrappedID3D12CommandQueue::ExecuteCommandLists(
 
   SAFE_DELETE_ARRAY(unwrapped);
 
-  bool capframe = false;
-  set<ResourceId> refdIDs;
-
-  for(UINT i = 0; i < NumCommandLists; i++)
+  if(m_State >= WRITING)
   {
-    D3D12ResourceRecord *record = GetRecord(ppCommandLists[i]);
+    bool capframe = false;
+    set<ResourceId> refdIDs;
 
-    m_pDevice->ApplyBarriers(record->bakedCommands->cmdInfo->barriers);
-
-    // need to lock the whole section of code, not just the check on
-    // m_State, as we also need to make sure we don't check the state,
-    // start marking dirty resources then while we're doing so the
-    // state becomes capframe.
-    // the next sections where we mark resources referenced and add
-    // the submit chunk to the frame record don't have to be protected.
-    // Only the decision of whether we're inframe or not, and marking
-    // dirty.
+    for(UINT i = 0; i < NumCommandLists; i++)
     {
-      SCOPED_LOCK(m_pDevice->GetCapTransitionLock());
-      if(m_State == WRITING_CAPFRAME)
-      {
-        for(auto it = record->bakedCommands->cmdInfo->dirtied.begin();
-            it != record->bakedCommands->cmdInfo->dirtied.end(); ++it)
-          GetResourceManager()->MarkPendingDirty(*it);
+      D3D12ResourceRecord *record = GetRecord(ppCommandLists[i]);
 
-        capframe = true;
-      }
-      else
+      m_pDevice->ApplyBarriers(record->bakedCommands->cmdInfo->barriers);
+
+      // need to lock the whole section of code, not just the check on
+      // m_State, as we also need to make sure we don't check the state,
+      // start marking dirty resources then while we're doing so the
+      // state becomes capframe.
+      // the next sections where we mark resources referenced and add
+      // the submit chunk to the frame record don't have to be protected.
+      // Only the decision of whether we're inframe or not, and marking
+      // dirty.
       {
-        for(auto it = record->bakedCommands->cmdInfo->dirtied.begin();
-            it != record->bakedCommands->cmdInfo->dirtied.end(); ++it)
-          GetResourceManager()->MarkDirtyResource(*it);
+        SCOPED_LOCK(m_pDevice->GetCapTransitionLock());
+        if(m_State == WRITING_CAPFRAME)
+        {
+          for(auto it = record->bakedCommands->cmdInfo->dirtied.begin();
+              it != record->bakedCommands->cmdInfo->dirtied.end(); ++it)
+            GetResourceManager()->MarkPendingDirty(*it);
+
+          capframe = true;
+        }
+        else
+        {
+          for(auto it = record->bakedCommands->cmdInfo->dirtied.begin();
+              it != record->bakedCommands->cmdInfo->dirtied.end(); ++it)
+            GetResourceManager()->MarkDirtyResource(*it);
+        }
       }
+
+      if(capframe)
+      {
+        // pull in frame refs from this baked command buffer
+        record->bakedCommands->AddResourceReferences(GetResourceManager());
+        record->bakedCommands->AddReferencedIDs(refdIDs);
+
+        // ref the parent command buffer by itself, this will pull in the cmd buffer pool
+        GetResourceManager()->MarkResourceFrameReferenced(record->GetResourceID(), eFrameRef_Read);
+
+        // reference all executed bundles as well
+        for(size_t b = 0; b < record->bakedCommands->cmdInfo->bundles.size(); b++)
+        {
+          record->bakedCommands->cmdInfo->bundles[b]->bakedCommands->AddResourceReferences(
+              GetResourceManager());
+          record->bakedCommands->cmdInfo->bundles[b]->bakedCommands->AddReferencedIDs(refdIDs);
+          GetResourceManager()->MarkResourceFrameReferenced(
+              record->bakedCommands->cmdInfo->bundles[b]->GetResourceID(), eFrameRef_Read);
+
+          record->bakedCommands->cmdInfo->bundles[b]->bakedCommands->AddRef();
+        }
+
+        {
+          m_CmdListRecords.push_back(record->bakedCommands);
+          for(size_t sub = 0; sub < record->bakedCommands->cmdInfo->bundles.size(); sub++)
+            m_CmdListRecords.push_back(record->bakedCommands->cmdInfo->bundles[sub]->bakedCommands);
+        }
+
+        record->bakedCommands->AddRef();
+      }
+
+      record->cmdInfo->dirtied.clear();
     }
 
     if(capframe)
     {
-      // pull in frame refs from this baked command buffer
-      record->bakedCommands->AddResourceReferences(GetResourceManager());
-      record->bakedCommands->AddReferencedIDs(refdIDs);
-
-      // ref the parent command buffer by itself, this will pull in the cmd buffer pool
-      GetResourceManager()->MarkResourceFrameReferenced(record->GetResourceID(), eFrameRef_Read);
-
-      // reference all executed bundles as well
-      for(size_t b = 0; b < record->bakedCommands->cmdInfo->bundles.size(); b++)
+      // flush coherent maps
+      for(UINT i = 0; i < NumCommandLists; i++)
       {
-        record->bakedCommands->cmdInfo->bundles[b]->bakedCommands->AddResourceReferences(
-            GetResourceManager());
-        record->bakedCommands->cmdInfo->bundles[b]->bakedCommands->AddReferencedIDs(refdIDs);
-        GetResourceManager()->MarkResourceFrameReferenced(
-            record->bakedCommands->cmdInfo->bundles[b]->GetResourceID(), eFrameRef_Read);
+        SCOPED_SERIALISE_CONTEXT(EXECUTE_CMD_LISTS);
+        Serialise_ExecuteCommandLists(1, ppCommandLists + i);
 
-        record->bakedCommands->cmdInfo->bundles[b]->bakedCommands->AddRef();
+        m_QueueRecord->AddChunk(scope.Get());
       }
-
-      {
-        m_CmdListRecords.push_back(record->bakedCommands);
-        for(size_t sub = 0; sub < record->bakedCommands->cmdInfo->bundles.size(); sub++)
-          m_CmdListRecords.push_back(record->bakedCommands->cmdInfo->bundles[sub]->bakedCommands);
-      }
-
-      record->bakedCommands->AddRef();
-    }
-
-    record->cmdInfo->dirtied.clear();
-  }
-
-  if(capframe)
-  {
-    // flush coherent maps
-    for(UINT i = 0; i < NumCommandLists; i++)
-    {
-      SCOPED_SERIALISE_CONTEXT(EXECUTE_CMD_LISTS);
-      Serialise_ExecuteCommandLists(1, ppCommandLists + i);
-
-      m_QueueRecord->AddChunk(scope.Get());
     }
   }
 }

renderdoc/driver/d3d12/d3d12_commands.cpp

@@ -366,7 +366,8 @@ void WrappedID3D12CommandQueue::ReplayLog(LogState readType, uint32_t startEvent
 
   m_pDevice->Serialise_BeginCaptureFrame(!partial);
 
-  m_pDevice->GPUSync();
+  m_pDevice->ExecuteLists();
+  m_pDevice->FlushLists();
 
   m_pSerialiser->PopContext(header);
 
@@ -440,7 +441,8 @@ void WrappedID3D12CommandQueue::ReplayLog(LogState readType, uint32_t startEvent
     }
   }
 
-  m_pDevice->GPUSync();
+  m_pDevice->ExecuteLists();
+  m_pDevice->FlushLists(true);
 
   if(m_State == READING)
   {

renderdoc/driver/d3d12/d3d12_debug.cpp

@@ -272,7 +272,8 @@ bool D3D12DebugManager::CheckResizeOutputWindow(uint64_t id)
     outw.width = w;
     outw.height = h;
 
-    m_WrappedDevice->GPUSync();
+    m_WrappedDevice->ExecuteLists();
+    m_WrappedDevice->FlushLists(true);
 
     if(outw.width > 0 && outw.height > 0)
     {
@@ -327,16 +328,11 @@ void D3D12DebugManager::ClearOutputWindowColour(uint64_t id, float col[4])
   if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
     return;
 
-  m_WrappedDevice->GetList()->Reset(m_WrappedDevice->GetAlloc(), NULL);
+  ID3D12GraphicsCommandList *list = m_WrappedDevice->GetNewList();
 
-  m_WrappedDevice->GetList()->ClearRenderTargetView(Unwrap(m_OutputWindows[id].rtv), col, 0, NULL);
+  list->ClearRenderTargetView(m_OutputWindows[id].rtv, col, 0, NULL);
 
-  m_WrappedDevice->GetList()->Close();
-
-  ID3D12CommandList *list = (ID3D12CommandList *)m_WrappedDevice->GetList();
-  m_WrappedDevice->GetQueue()->GetReal()->ExecuteCommandLists(1, &list);
-
-  m_WrappedDevice->GPUSync();
+  list->Close();
 }
 
 void D3D12DebugManager::ClearOutputWindowDepth(uint64_t id, float depth, uint8_t stencil)
@@ -344,18 +340,13 @@ void D3D12DebugManager::ClearOutputWindowDepth(uint64_t id, float depth, uint8_t
   if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end())
     return;
 
-  m_WrappedDevice->GetList()->Reset(m_WrappedDevice->GetAlloc(), NULL);
+  ID3D12GraphicsCommandList *list = m_WrappedDevice->GetNewList();
 
-  m_WrappedDevice->GetList()->ClearDepthStencilView(
-      Unwrap(m_OutputWindows[id].dsv), D3D12_CLEAR_FLAG_DEPTH | D3D12_CLEAR_FLAG_STENCIL, depth,
-      stencil, 0, NULL);
+  list->ClearDepthStencilView(m_OutputWindows[id].dsv,
+                              D3D12_CLEAR_FLAG_DEPTH | D3D12_CLEAR_FLAG_STENCIL, depth, stencil, 0,
+                              NULL);
 
-  m_WrappedDevice->GetList()->Close();
-
-  ID3D12CommandList *list = (ID3D12CommandList *)m_WrappedDevice->GetList();
-  m_WrappedDevice->GetQueue()->GetReal()->ExecuteCommandLists(1, &list);
-
-  m_WrappedDevice->GPUSync();
+  list->Close();
 }
 
 void D3D12DebugManager::BindOutputWindow(uint64_t id, bool depth)
@@ -365,11 +356,12 @@ void D3D12DebugManager::BindOutputWindow(uint64_t id, bool depth)
 
   OutputWindow &outw = m_OutputWindows[id];
 
+  m_CurrentOutputWindow = id;
+
   if(outw.bb[0] == NULL)
     return;
 
-  m_width = (int32_t)outw.width;
-  m_height = (int32_t)outw.height;
+  SetOutputDimensions(outw.width, outw.height);
 }
 
 bool D3D12DebugManager::IsOutputWindowVisible(uint64_t id)
@@ -393,40 +385,36 @@ void D3D12DebugManager::FlipOutputWindow(uint64_t id)
   D3D12_RESOURCE_BARRIER barriers[2];
   RDCEraseEl(barriers);
 
-  barriers[0].Transition.pResource = Unwrap(outw.col);
+  barriers[0].Transition.pResource = outw.col;
   barriers[0].Transition.StateBefore = D3D12_RESOURCE_STATE_RENDER_TARGET;
   barriers[0].Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_SOURCE;
 
-  barriers[1].Transition.pResource = Unwrap(outw.bb[outw.bbIdx]);
+  barriers[1].Transition.pResource = outw.bb[outw.bbIdx];
   barriers[1].Transition.StateBefore = D3D12_RESOURCE_STATE_PRESENT;
   barriers[1].Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_DEST;
 
-  m_WrappedDevice->GetList()->Reset(m_WrappedDevice->GetAlloc(), NULL);
+  ID3D12GraphicsCommandList *list = m_WrappedDevice->GetNewList();
 
   // transition colour to copy source, backbuffer to copy test
-  m_WrappedDevice->GetList()->ResourceBarrier(2, barriers);
+  list->ResourceBarrier(2, barriers);
 
   // resolve or copy from colour to backbuffer
   if(outw.depth)
-    m_WrappedDevice->GetList()->ResolveSubresource(barriers[1].Transition.pResource, 0,
-                                                   barriers[0].Transition.pResource, 0,
-                                                   DXGI_FORMAT_R8G8B8A8_UNORM_SRGB);
+    list->ResolveSubresource(barriers[1].Transition.pResource, 0, barriers[0].Transition.pResource,
+                             0, DXGI_FORMAT_R8G8B8A8_UNORM_SRGB);
   else
-    m_WrappedDevice->GetList()->CopyResource(barriers[1].Transition.pResource,
-                                             barriers[0].Transition.pResource);
+    list->CopyResource(barriers[1].Transition.pResource, barriers[0].Transition.pResource);
 
   std::swap(barriers[0].Transition.StateBefore, barriers[0].Transition.StateAfter);
   std::swap(barriers[1].Transition.StateBefore, barriers[1].Transition.StateAfter);
 
   // transition colour back to render target, and backbuffer back to present
-  m_WrappedDevice->GetList()->ResourceBarrier(2, barriers);
+  list->ResourceBarrier(2, barriers);
 
-  m_WrappedDevice->GetList()->Close();
+  list->Close();
 
-  ID3D12CommandList *list = (ID3D12CommandList *)m_WrappedDevice->GetList();
-  m_WrappedDevice->GetQueue()->GetReal()->ExecuteCommandLists(1, &list);
-
-  m_WrappedDevice->GPUSync();
+  m_WrappedDevice->ExecuteLists();
+  m_WrappedDevice->FlushLists();
 
   outw.swap->Present(0, 0);
 

renderdoc/driver/d3d12/d3d12_device.cpp

@@ -1220,15 +1220,11 @@ void WrappedID3D12Device::SetResourceName(ID3D12DeviceChild *res, const char *na
 
 void WrappedID3D12Device::CreateInternalResources()
 {
-  m_pDevice->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT,
-                                    __uuidof(ID3D12CommandAllocator), (void **)&m_Alloc);
-  m_pDevice->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, m_Alloc, NULL,
-                               __uuidof(ID3D12GraphicsCommandList), (void **)&m_List);
-  m_pDevice->CreateFence(0, D3D12_FENCE_FLAG_NONE, __uuidof(ID3D12Fence), (void **)&m_GPUSyncFence);
+  CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, __uuidof(ID3D12CommandAllocator),
+                         (void **)&m_Alloc);
+  CreateFence(0, D3D12_FENCE_FLAG_NONE, __uuidof(ID3D12Fence), (void **)&m_GPUSyncFence);
   m_GPUSyncHandle = ::CreateEvent(NULL, FALSE, FALSE, NULL);
 
-  m_List->Close();
-
   m_GPUSyncCounter = 0;
 }
 
@@ -1237,8 +1233,13 @@ void WrappedID3D12Device::DestroyInternalResources()
   if(m_GPUSyncHandle == NULL)
     return;
 
+  ExecuteLists();
+  FlushLists(true);
+
+  for(size_t i = 0; i < m_InternalCmds.pendingcmds.size(); i++)
+    SAFE_RELEASE(m_InternalCmds.pendingcmds[i]);
+
   SAFE_RELEASE(m_Alloc);
-  SAFE_RELEASE(m_List);
   SAFE_RELEASE(m_GPUSyncFence);
   CloseHandle(m_GPUSyncHandle);
 }
@@ -1247,11 +1248,68 @@ void WrappedID3D12Device::GPUSync()
 {
   m_GPUSyncCounter++;
 
-  m_Queue->GetReal()->Signal(m_GPUSyncFence, m_GPUSyncCounter);
+  m_Queue->Signal(m_GPUSyncFence, m_GPUSyncCounter);
   m_GPUSyncFence->SetEventOnCompletion(m_GPUSyncCounter, m_GPUSyncHandle);
   WaitForSingleObject(m_GPUSyncHandle, 2000);
 }
 
+ID3D12GraphicsCommandList *WrappedID3D12Device::GetNewList()
+{
+  ID3D12GraphicsCommandList *ret;
+
+  if(!m_InternalCmds.freecmds.empty())
+  {
+    ret = m_InternalCmds.freecmds.back();
+    m_InternalCmds.freecmds.pop_back();
+
+    ret->Reset(m_Alloc, NULL);
+  }
+  else
+  {
+    HRESULT hr = CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, m_Alloc, NULL,
+                                   __uuidof(ID3D12GraphicsCommandList), (void **)&ret);
+
+    RDCASSERTEQUAL(hr, S_OK);
+  }
+
+  m_InternalCmds.pendingcmds.push_back(ret);
+
+  return ret;
+}
+
+void WrappedID3D12Device::ExecuteLists()
+{
+  // nothing to do
+  if(m_InternalCmds.pendingcmds.empty())
+    return;
+
+  vector<ID3D12CommandList *> cmds;
+  cmds.resize(m_InternalCmds.pendingcmds.size());
+  for(size_t i = 0; i < cmds.size(); i++)
+    cmds[i] = m_InternalCmds.pendingcmds[i];
+
+  GetQueue()->ExecuteCommandLists((UINT)cmds.size(), &cmds[0]);
+
+  m_InternalCmds.submittedcmds.insert(m_InternalCmds.submittedcmds.end(),
+                                      m_InternalCmds.pendingcmds.begin(),
+                                      m_InternalCmds.pendingcmds.end());
+  m_InternalCmds.pendingcmds.clear();
+}
+
+void WrappedID3D12Device::FlushLists(bool forceSync)
+{
+  if(!m_InternalCmds.submittedcmds.empty() || forceSync)
+  {
+    GPUSync();
+
+    if(!m_InternalCmds.submittedcmds.empty())
+      m_InternalCmds.freecmds.insert(m_InternalCmds.freecmds.end(),
+                                     m_InternalCmds.submittedcmds.begin(),
+                                     m_InternalCmds.submittedcmds.end());
+    m_InternalCmds.submittedcmds.clear();
+  }
+}
+
 void WrappedID3D12Device::SetLogFile(const char *logfile)
 {
   m_pSerialiser = new Serialiser(logfile, Serialiser::READING, false);
@@ -1448,7 +1506,8 @@ void WrappedID3D12Device::ReplayLog(uint32_t startEventID, uint32_t endEventID,
     GetResourceManager()->ApplyInitialContents();
     GetResourceManager()->ReleaseInFrameResources();
 
-    GPUSync();
+    ExecuteLists();
+    FlushLists(true);
   }
 
   m_State = EXECUTING;

renderdoc/driver/d3d12/d3d12_device.h

@@ -338,10 +338,31 @@ public:
   D3D12Replay *GetReplay() { return &m_Replay; }
   WrappedID3D12CommandQueue *GetQueue() { return m_Queue; }
   ID3D12CommandAllocator *GetAlloc() { return m_Alloc; }
-  ID3D12GraphicsCommandList *GetList() { return m_List; }
-  void GPUSync();
   void ApplyBarriers(vector<D3D12_RESOURCE_BARRIER> &barriers);
 
+  struct
+  {
+    void Reset()
+    {
+      freecmds.clear();
+      pendingcmds.clear();
+      submittedcmds.clear();
+    }
+
+    vector<ID3D12GraphicsCommandList *> freecmds;
+    // -> GetNextCmd() ->
+    vector<ID3D12GraphicsCommandList *> pendingcmds;
+    // -> ExecuteLists() ->
+    vector<ID3D12GraphicsCommandList *> submittedcmds;
+    // -> FlushLists()--------back to freecmds--------^
+  } m_InternalCmds;
+
+  ID3D12GraphicsCommandList *GetNewList();
+  void ExecuteLists();
+  void FlushLists(bool forceSync = false);
+
+  void GPUSync();
+
   void StartFrameCapture(void *dev, void *wnd);
   bool EndFrameCapture(void *dev, void *wnd);
 

renderdoc/driver/d3d12/d3d12_manager.cpp

@@ -23,6 +23,7 @@
  ******************************************************************************/
 
 #include "d3d12_manager.h"
+#include "d3d12_command_list.h"
 #include "d3d12_command_queue.h"
 #include "d3d12_device.h"
 #include "d3d12_resources.h"
@@ -349,14 +350,11 @@ bool D3D12ResourceManager::Prepare_InitialState(ID3D12DeviceChild *res)
 
       if(SUCCEEDED(hr))
       {
-        m_Device->GetList()->Reset(m_Device->GetAlloc(), NULL);
+        ID3D12GraphicsCommandList *list = Unwrap(m_Device->GetNewList());
 
-        m_Device->GetList()->CopyResource(copyDst, Unwrap(r));
+        list->CopyResource(copyDst, Unwrap(r));
 
-        m_Device->GetList()->Close();
-
-        ID3D12CommandList *list = (ID3D12CommandList *)m_Device->GetList();
-        m_Device->GetQueue()->GetReal()->ExecuteCommandLists(1, &list);
+        list->Close();
       }
       else
       {
@@ -405,7 +403,8 @@ bool D3D12ResourceManager::Serialise_InitialState(ResourceId resid, ID3D12Device
 
       if(desc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER)
       {
-        m_Device->GPUSync();
+        m_Device->ExecuteLists();
+        m_Device->FlushLists();
 
         ID3D12Resource *copySrc = (ID3D12Resource *)initContents.resource;
 
@@ -654,7 +653,7 @@ void D3D12ResourceManager::Apply_InitialState(ID3D12DeviceChild *live, InitialCo
       }
       else
       {
-        m_Device->GetList()->Reset(m_Device->GetAlloc(), NULL);
+        ID3D12GraphicsCommandList *list = Unwrap(m_Device->GetNewList());
 
         D3D12_RESOURCE_BARRIER barrier;
         barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
@@ -666,19 +665,16 @@ void D3D12ResourceManager::Apply_InitialState(ID3D12DeviceChild *live, InitialCo
 
         // transition to copy dest
         if(barrier.Transition.StateBefore != barrier.Transition.StateAfter)
-          m_Device->GetList()->ResourceBarrier(1, &barrier);
+          list->ResourceBarrier(1, &barrier);
 
-        m_Device->GetList()->CopyBufferRegion(copyDst, 0, copySrc, 0, copySrc->GetDesc().Width);
+        list->CopyBufferRegion(copyDst, 0, copySrc, 0, copySrc->GetDesc().Width);
 
         // transition back to whatever it was before
         std::swap(barrier.Transition.StateBefore, barrier.Transition.StateAfter);
         if(barrier.Transition.StateBefore != barrier.Transition.StateAfter)
-          m_Device->GetList()->ResourceBarrier(1, &barrier);
+          list->ResourceBarrier(1, &barrier);
 
-        m_Device->GetList()->Close();
-
-        ID3D12CommandList *list = (ID3D12CommandList *)m_Device->GetList();
-        m_Device->GetQueue()->GetReal()->ExecuteCommandLists(1, &list);
+        list->Close();
       }
     }
     else

renderdoc/driver/vulkan/vk_core.h

@@ -288,26 +288,26 @@ private:
       pendingcmds.clear();
       submittedcmds.clear();
 
-      freecmds.clear();
-      pendingcmds.clear();
-      submittedcmds.clear();
+      freesems.clear();
+      pendingsems.clear();
+      submittedsems.clear();
     }
 
     VkCommandPool cmdpool;    // the command pool used for allocating our own command buffers
 
-    vector<VkCommandBuffer> freecmds;    // <
-    // -> GetNextCmd() ->                   // |
-    vector<VkCommandBuffer> pendingcmds;    // |
-    // -> SubmitCmds() ->                      |
-    vector<VkCommandBuffer> submittedcmds;    // |
-    // -> FlushQ() ----------------------------^
+    vector<VkCommandBuffer> freecmds;
+    // -> GetNextCmd() ->
+    vector<VkCommandBuffer> pendingcmds;
+    // -> SubmitCmds() ->
+    vector<VkCommandBuffer> submittedcmds;
+    // -> FlushQ() ------back to freecmds------^
 
-    vector<VkSemaphore> freesems;    // <
-    // -> GetNextSemaphore() ->        // |
-    vector<VkSemaphore> pendingsems;    // |
-    // -> SubmitSemaphores() ->           |
-    vector<VkSemaphore> submittedsems;    // |
-                                          // -> FlushQ() -----------------------^
+    vector<VkSemaphore> freesems;
+    // -> GetNextSemaphore() ->
+    vector<VkSemaphore> pendingsems;
+    // -> SubmitSemaphores() ->
+    vector<VkSemaphore> submittedsems;
+    // -> FlushQ() ----back to freesems-------^
   } m_InternalCmds;
 
   vector<VkDeviceMemory> m_CleanupMems;