Don't ever wait for idle when rendering overlay in vulkan

* Instead of waiting for idle, we allocate a command buffer per swapchain image
  to render the text overlay and use semaphores and fences to properly
  synchronise with other GPU work ongoing.

renderdoc/driver/vulkan/vk_core.cpp

@@ -373,7 +373,7 @@ VkCommandBuffer WrappedVulkan::GetExtQueueCmd(uint32_t queueFamilyIdx) const
     return VK_NULL_HANDLE;
   }
 
-  VkCommandBuffer buf = m_ExternalQueues[queueFamilyIdx].buffer;
+  VkCommandBuffer buf = m_ExternalQueues[queueFamilyIdx].ring[0].acquire;
 
   ObjDisp(buf)->ResetCommandBuffer(Unwrap(buf), 0);
 
@@ -388,7 +388,7 @@ void WrappedVulkan::SubmitAndFlushExtQueue(uint32_t queueFamilyIdx) const
     return;
   }
 
-  VkCommandBuffer buf = Unwrap(m_ExternalQueues[queueFamilyIdx].buffer);
+  VkCommandBuffer buf = Unwrap(m_ExternalQueues[queueFamilyIdx].ring[0].acquire);
 
   VkSubmitInfo submitInfo = {
       VK_STRUCTURE_TYPE_SUBMIT_INFO,

renderdoc/driver/vulkan/vk_core.h

@@ -450,7 +450,22 @@ private:
   {
     VkQueue queue = VK_NULL_HANDLE;
     VkCommandPool pool = VK_NULL_HANDLE;
-    VkCommandBuffer buffer = VK_NULL_HANDLE;
+
+    struct
+    {
+      VkCommandBuffer acquire = VK_NULL_HANDLE;
+      VkCommandBuffer release = VK_NULL_HANDLE;
+      VkSemaphore fromext = VK_NULL_HANDLE;
+      VkSemaphore toext = VK_NULL_HANDLE;
+      VkFence fence = VK_NULL_HANDLE;
+    } ring[4];
+    uint32_t ringIndex = 0;
+    uint32_t GetNextIdx()
+    {
+      uint32_t ret = ringIndex;
+      ringIndex = (ringIndex + 1) % ARRAY_COUNT(ring);
+      return ret;
+    }
   };
   rdcarray<ExternalQueue> m_ExternalQueues;
 
@@ -938,9 +953,6 @@ private:
                                                       int32_t messageCode, const char *pLayerPrefix,
                                                       const char *pMessage, void *pUserData);
   void AddFrameTerminator(uint64_t queueMarkerTag);
-  void SubmitExtQBarriers(const std::map<uint32_t, rdcarray<VkImageMemoryBarrier>> &extQBarriers);
-  void SubmitExtQBarriers(uint32_t queueFamilyIndex,
-                          const rdcarray<VkImageMemoryBarrier> &queueFamilyBarriers);
 
   VkResourceRecord *RegisterSurface(WindowingSystem system, void *handle);
 

renderdoc/driver/vulkan/vk_initstate.cpp

@@ -1334,42 +1334,6 @@ std::map<uint32_t, rdcarray<VkImageMemoryBarrier> > GetExtQBarriers(
   return extQBarriers;
 }
 
-void WrappedVulkan::SubmitExtQBarriers(uint32_t queueFamilyIndex,
-                                       const rdcarray<VkImageMemoryBarrier> &queueFamilyBarriers)
-{
-  if(queueFamilyBarriers.empty())
-    return;
-
-  if(queueFamilyIndex == ~0U)
-    queueFamilyIndex = queueFamilyBarriers[0].dstQueueFamilyIndex;
-
-  VkCommandBuffer extQCmd = GetExtQueueCmd(queueFamilyIndex);
-
-  VkCommandBufferBeginInfo beginInfo = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, NULL,
-                                        VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
-
-  VkResult vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
-  RDCASSERTEQUAL(vkr, VK_SUCCESS);
-
-  DoPipelineBarrier(extQCmd, queueFamilyBarriers.size(), queueFamilyBarriers.data());
-  vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
-  RDCASSERTEQUAL(vkr, VK_SUCCESS);
-
-  SubmitAndFlushExtQueue(queueFamilyIndex);
-}
-
-void WrappedVulkan::SubmitExtQBarriers(
-    const std::map<uint32_t, rdcarray<VkImageMemoryBarrier> > &extQBarriers)
-{
-  for(auto extQBarrierIt = extQBarriers.begin(); extQBarrierIt != extQBarriers.end(); ++extQBarrierIt)
-  {
-    uint32_t queueFamilyIndex = extQBarrierIt->first;
-    const rdcarray<VkImageMemoryBarrier> &queueFamilyBarriers = extQBarrierIt->second;
-
-    SubmitExtQBarriers(queueFamilyIndex, queueFamilyBarriers);
-  }
-}
-
 void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, const VkInitialContents &initial)
 {
   VkResourceType type = initial.type;

renderdoc/driver/vulkan/vk_resources.h

@@ -961,6 +961,9 @@ struct SwapchainInfo
 
     VkImageView view;
     VkFramebuffer fb;
+
+    VkCommandBuffer cmd;
+    VkSemaphore overlaydone;
   };
   rdcarray<SwapImage> images;
   PresentInfo lastPresent;

renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp

@@ -803,6 +803,13 @@ void WrappedVulkan::Shutdown()
   SubmitSemaphores();
   FlushQ();
 
+  // idle the device as well so that external queues are idle.
+  if(m_Device)
+  {
+    VkResult vkr = ObjDisp(m_Device)->DeviceWaitIdle(Unwrap(m_Device));
+    RDCASSERTEQUAL(vkr, VK_SUCCESS);
+  }
+
   // destroy any events we created for waiting on
   for(size_t i = 0; i < m_PersistentEvents.size(); i++)
     ObjDisp(GetDev())->DestroyEvent(Unwrap(GetDev()), m_PersistentEvents[i], NULL);
@@ -833,9 +840,25 @@ void WrappedVulkan::Shutdown()
 
   for(size_t i = 0; i < m_ExternalQueues.size(); i++)
   {
-    if(m_ExternalQueues[i].buffer != VK_NULL_HANDLE)
+    if(m_ExternalQueues[i].pool != VK_NULL_HANDLE)
     {
-      GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].buffer);
+      for(size_t x = 0; x < ARRAY_COUNT(m_ExternalQueues[i].ring); x++)
+      {
+        GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].acquire);
+        GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].release);
+
+        ObjDisp(m_Device)->DestroySemaphore(Unwrap(m_Device),
+                                            Unwrap(m_ExternalQueues[i].ring[x].fromext), NULL);
+        GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].fromext);
+
+        ObjDisp(m_Device)->DestroySemaphore(Unwrap(m_Device),
+                                            Unwrap(m_ExternalQueues[i].ring[x].toext), NULL);
+        GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].toext);
+
+        ObjDisp(m_Device)->DestroyFence(Unwrap(m_Device), Unwrap(m_ExternalQueues[i].ring[x].fence),
+                                        NULL);
+        GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].fence);
+      }
 
       ObjDisp(m_Device)->DestroyCommandPool(Unwrap(m_Device), Unwrap(m_ExternalQueues[i].pool), NULL);
       GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].pool);
@@ -3138,16 +3161,52 @@ bool WrappedVulkan::Serialise_vkCreateDevice(SerialiserType &ser, VkPhysicalDevi
           1,
       };
 
-      vkr = ObjDisp(device)->AllocateCommandBuffers(Unwrap(device), &cmdInfo,
-                                                    &m_ExternalQueues[qidx].buffer);
-      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+      VkFenceCreateInfo fenceInfo = {VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, NULL,
+                                     VK_FENCE_CREATE_SIGNALED_BIT};
+      VkSemaphoreCreateInfo semInfo = {VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO};
 
-      if(m_SetDeviceLoaderData)
-        m_SetDeviceLoaderData(device, m_ExternalQueues[qidx].buffer);
-      else
-        SetDispatchTableOverMagicNumber(device, m_ExternalQueues[qidx].buffer);
+      for(size_t x = 0; x < ARRAY_COUNT(m_ExternalQueues[i].ring); x++)
+      {
+        vkr = ObjDisp(device)->AllocateCommandBuffers(Unwrap(device), &cmdInfo,
+                                                      &m_ExternalQueues[qidx].ring[x].acquire);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
-      GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].buffer);
+        if(m_SetDeviceLoaderData)
+          m_SetDeviceLoaderData(device, m_ExternalQueues[qidx].ring[x].acquire);
+        else
+          SetDispatchTableOverMagicNumber(device, m_ExternalQueues[qidx].ring[x].acquire);
+
+        GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].acquire);
+
+        vkr = ObjDisp(device)->AllocateCommandBuffers(Unwrap(device), &cmdInfo,
+                                                      &m_ExternalQueues[qidx].ring[x].release);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+        if(m_SetDeviceLoaderData)
+          m_SetDeviceLoaderData(device, m_ExternalQueues[qidx].ring[x].release);
+        else
+          SetDispatchTableOverMagicNumber(device, m_ExternalQueues[qidx].ring[x].release);
+
+        GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].release);
+
+        vkr = ObjDisp(device)->CreateSemaphore(Unwrap(device), &semInfo, NULL,
+                                               &m_ExternalQueues[qidx].ring[x].fromext);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+        GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].fromext);
+
+        vkr = ObjDisp(device)->CreateSemaphore(Unwrap(device), &semInfo, NULL,
+                                               &m_ExternalQueues[qidx].ring[x].toext);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+        GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].toext);
+
+        vkr = ObjDisp(device)->CreateFence(Unwrap(device), &fenceInfo, NULL,
+                                           &m_ExternalQueues[qidx].ring[x].fence);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+        GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].fence);
+      }
     }
 
     ObjDisp(physicalDevice)
@@ -3537,16 +3596,52 @@ VkResult WrappedVulkan::vkCreateDevice(VkPhysicalDevice physicalDevice,
           1,
       };
 
-      vkr = ObjDisp(device)->AllocateCommandBuffers(Unwrap(device), &cmdInfo,
-                                                    &m_ExternalQueues[qidx].buffer);
-      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+      VkFenceCreateInfo fenceInfo = {VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, NULL,
+                                     VK_FENCE_CREATE_SIGNALED_BIT};
+      VkSemaphoreCreateInfo semInfo = {VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO};
 
-      if(m_SetDeviceLoaderData)
-        m_SetDeviceLoaderData(device, m_ExternalQueues[qidx].buffer);
-      else
-        SetDispatchTableOverMagicNumber(device, m_ExternalQueues[qidx].buffer);
+      for(size_t x = 0; x < ARRAY_COUNT(m_ExternalQueues[i].ring); x++)
+      {
+        vkr = ObjDisp(device)->AllocateCommandBuffers(Unwrap(device), &cmdInfo,
+                                                      &m_ExternalQueues[qidx].ring[x].acquire);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
-      GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].buffer);
+        if(m_SetDeviceLoaderData)
+          m_SetDeviceLoaderData(device, m_ExternalQueues[qidx].ring[x].acquire);
+        else
+          SetDispatchTableOverMagicNumber(device, m_ExternalQueues[qidx].ring[x].acquire);
+
+        GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].acquire);
+
+        vkr = ObjDisp(device)->AllocateCommandBuffers(Unwrap(device), &cmdInfo,
+                                                      &m_ExternalQueues[qidx].ring[x].release);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+        if(m_SetDeviceLoaderData)
+          m_SetDeviceLoaderData(device, m_ExternalQueues[qidx].ring[x].release);
+        else
+          SetDispatchTableOverMagicNumber(device, m_ExternalQueues[qidx].ring[x].release);
+
+        GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].release);
+
+        vkr = ObjDisp(device)->CreateSemaphore(Unwrap(device), &semInfo, NULL,
+                                               &m_ExternalQueues[qidx].ring[x].fromext);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+        GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].fromext);
+
+        vkr = ObjDisp(device)->CreateSemaphore(Unwrap(device), &semInfo, NULL,
+                                               &m_ExternalQueues[qidx].ring[x].toext);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+        GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].toext);
+
+        vkr = ObjDisp(device)->CreateFence(Unwrap(device), &fenceInfo, NULL,
+                                           &m_ExternalQueues[qidx].ring[x].fence);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+        GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].fence);
+      }
     }
 
     ObjDisp(physicalDevice)
@@ -3586,6 +3681,10 @@ void WrappedVulkan::vkDestroyDevice(VkDevice device, const VkAllocationCallbacks
   SubmitSemaphores();
   FlushQ();
 
+  // idle the device as well so that external queues are idle.
+  VkResult vkr = ObjDisp(m_Device)->DeviceWaitIdle(Unwrap(m_Device));
+  RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
   // MULTIDEVICE this function will need to check if the device is the one we
   // used for debugmanager/cmd pool etc, and only remove child queues and
   // resources (instead of doing full resource manager shutdown).
@@ -3621,9 +3720,25 @@ void WrappedVulkan::vkDestroyDevice(VkDevice device, const VkAllocationCallbacks
 
   for(size_t i = 0; i < m_ExternalQueues.size(); i++)
   {
-    if(m_ExternalQueues[i].buffer != VK_NULL_HANDLE)
+    if(m_ExternalQueues[i].pool != VK_NULL_HANDLE)
     {
-      GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].buffer);
+      for(size_t x = 0; x < ARRAY_COUNT(m_ExternalQueues[i].ring); x++)
+      {
+        GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].acquire);
+        GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].release);
+
+        ObjDisp(m_Device)->DestroySemaphore(Unwrap(m_Device),
+                                            Unwrap(m_ExternalQueues[i].ring[x].fromext), NULL);
+        GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].fromext);
+
+        ObjDisp(m_Device)->DestroySemaphore(Unwrap(m_Device),
+                                            Unwrap(m_ExternalQueues[i].ring[x].toext), NULL);
+        GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].toext);
+
+        ObjDisp(m_Device)->DestroyFence(Unwrap(m_Device), Unwrap(m_ExternalQueues[i].ring[x].fence),
+                                        NULL);
+        GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].fence);
+      }
 
       ObjDisp(m_Device)->DestroyCommandPool(Unwrap(m_Device), Unwrap(m_ExternalQueues[i].pool), NULL);
       GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].pool);

renderdoc/driver/vulkan/wrappers/vk_misc_funcs.cpp

@@ -243,6 +243,8 @@ void WrappedVulkan::vkDestroySwapchainKHR(VkDevice device, VkSwapchainKHR obj,
   {
     SwapchainInfo &info = *GetRecord(obj)->swapInfo;
 
+    ObjDisp(device)->DeviceWaitIdle(Unwrap(device));
+
     VkRenderPass unwrappedRP = Unwrap(info.rp);
     GetResourceManager()->ReleaseWrappedResource(info.rp, true);
     ObjDisp(device)->DestroyRenderPass(Unwrap(device), unwrappedRP, NULL);
@@ -251,12 +253,18 @@ void WrappedVulkan::vkDestroySwapchainKHR(VkDevice device, VkSwapchainKHR obj,
     {
       VkFramebuffer unwrappedFB = Unwrap(info.images[i].fb);
       VkImageView unwrappedView = Unwrap(info.images[i].view);
+      VkSemaphore unwrappedSem = Unwrap(info.images[i].overlaydone);
       GetResourceManager()->ReleaseWrappedResource(info.images[i].fb, true);
       // note, image doesn't have to be destroyed, just untracked
       GetResourceManager()->ReleaseWrappedResource(info.images[i].im, true);
       GetResourceManager()->ReleaseWrappedResource(info.images[i].view, true);
+      GetResourceManager()->ReleaseWrappedResource(info.images[i].overlaydone);
       ObjDisp(device)->DestroyFramebuffer(Unwrap(device), unwrappedFB, NULL);
       ObjDisp(device)->DestroyImageView(Unwrap(device), unwrappedView, NULL);
+      ObjDisp(device)->DestroySemaphore(Unwrap(device), unwrappedSem, NULL);
+
+      // return the command buffers to the pool
+      AddFreeCommandBuffer(info.images[i].cmd);
     }
   }
 

renderdoc/driver/vulkan/wrappers/vk_wsi_funcs.cpp

@@ -583,6 +583,21 @@ void WrappedVulkan::WrapAndProcessCreatedSwapchain(VkDevice device,
           state->isMemoryBound = true;
         }
 
+        // take a command buffer and steal it
+        swapImInfo.cmd = GetNextCmd();
+        RemovePendingCommandBuffer(swapImInfo.cmd);
+
+        {
+          VkSemaphoreCreateInfo semInfo = {VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO};
+
+          vkr = ObjDisp(device)->CreateSemaphore(Unwrap(device), &semInfo, NULL,
+                                                 &swapImInfo.overlaydone);
+          RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+          GetResourceManager()->WrapResource(Unwrap(device), swapImInfo.overlaydone);
+          GetResourceManager()->SetInternalResource(GetResID(swapImInfo.overlaydone));
+        }
+
         {
           VkImageViewCreateInfo info = {
               VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
@@ -714,8 +729,8 @@ VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR
   for(uint32_t i = 0; i < unwrappedInfo.waitSemaphoreCount; i++)
     unwrappedSems.push_back(Unwrap(unwrappedInfo.pWaitSemaphores[i]));
 
-  unwrappedInfo.pSwapchains = unwrappedInfo.swapchainCount ? &unwrappedSwaps[0] : NULL;
-  unwrappedInfo.pWaitSemaphores = unwrappedInfo.waitSemaphoreCount ? &unwrappedSems[0] : NULL;
+  unwrappedInfo.pSwapchains = unwrappedSwaps.data();
+  unwrappedInfo.pWaitSemaphores = unwrappedSems.data();
 
   // Don't support any extensions for present info
   const VkBaseInStructure *next = (const VkBaseInStructure *)pPresentInfo->pNext;
@@ -764,6 +779,8 @@ VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR
       VkRenderPass rp = swapInfo.rp;
       VkImage im = swapInfo.images[pPresentInfo->pImageIndices[0]].im;
       VkFramebuffer fb = swapInfo.images[pPresentInfo->pImageIndices[0]].fb;
+      VkCommandBuffer cmd = swapInfo.images[pPresentInfo->pImageIndices[0]].cmd;
+      VkSemaphore sem = swapInfo.images[pPresentInfo->pImageIndices[0]].overlaydone;
 
       VkResourceRecord *queueRecord = GetRecord(queue);
       uint32_t swapQueueIndex = queueRecord->queueFamilyIndex;
@@ -771,7 +788,7 @@ VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR
       VkDevDispatchTable *vt = ObjDisp(GetDev());
 
       TextPrintState textstate = {
-          GetNextCmd(),
+          cmd,
           rp,
           fb,
           RDCMAX(1U, swapInfo.imageInfo.extent.width),
@@ -779,10 +796,12 @@ VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR
           swapInfo.imageInfo.format,
       };
 
+      ObjDisp(cmd)->ResetCommandBuffer(Unwrap(cmd), 0);
+
       VkCommandBufferBeginInfo beginInfo = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, NULL,
                                             VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
 
-      VkResult vkr = vt->BeginCommandBuffer(Unwrap(textstate.cmd), &beginInfo);
+      VkResult vkr = vt->BeginCommandBuffer(Unwrap(cmd), &beginInfo);
       RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
       VkImageMemoryBarrier bbBarrier = {
@@ -804,11 +823,35 @@ VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR
       bbBarrier.srcAccessMask = VK_ACCESS_ALL_READ_BITS;
       bbBarrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
 
-      DoPipelineBarrier(textstate.cmd, 1, &bbBarrier);
+      DoPipelineBarrier(cmd, 1, &bbBarrier);
+
+      rdcarray<VkPipelineStageFlags> waitStage;
+      waitStage.fill(unwrappedSems.size(), VK_PIPELINE_STAGE_ALL_COMMANDS_BIT);
+
+      uint32_t ringIdx = 0;
+
+      VkSubmitInfo submitInfo = {VK_STRUCTURE_TYPE_SUBMIT_INFO};
+
+      // wait on the present's semaphores
+      submitInfo.pWaitDstStageMask = waitStage.data();
+      submitInfo.pWaitSemaphores = unwrappedSems.data();
+      submitInfo.waitSemaphoreCount = (uint32_t)unwrappedSems.size();
+
+      // and signal overlaydone
+      submitInfo.pSignalSemaphores = UnwrapPtr(sem);
+      submitInfo.signalSemaphoreCount = 1;
 
       if(swapQueueIndex != m_QueueFamilyIdx)
       {
-        VkCommandBuffer extQCmd = GetExtQueueCmd(swapQueueIndex);
+        ringIdx = m_ExternalQueues[swapQueueIndex].GetNextIdx();
+
+        VkCommandBuffer extQCmd = m_ExternalQueues[swapQueueIndex].ring[ringIdx].acquire;
+        VkFence fence = m_ExternalQueues[swapQueueIndex].ring[ringIdx].fence;
+
+        // wait for this queue ring to be free, but with a reasonably aggressive timeout (50ms). If
+        // this ring has never been used the fence is signalled on creation
+        vkr = vt->WaitForFences(Unwrap(m_Device), 1, UnwrapPtr(fence), VK_TRUE, 50000000);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
         vkr = vt->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
         RDCASSERTEQUAL(vkr, VK_SUCCESS);
@@ -817,7 +860,33 @@ VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR
 
         ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
 
-        SubmitAndFlushExtQueue(swapQueueIndex);
+        VkQueue q = m_ExternalQueues[swapQueueIndex].queue;
+
+        // wait on present semaphores as before, but on the proper (external) queue.
+
+        // submit the acquire command buffer (release from the external point of view)
+        submitInfo.commandBufferCount = 1;
+        submitInfo.pCommandBuffers = UnwrapPtr(extQCmd);
+
+        // signal the semaphore that we'll wait on for our overlay command buffer
+        submitInfo.signalSemaphoreCount = 1;
+        submitInfo.pSignalSemaphores =
+            UnwrapPtr(m_ExternalQueues[swapQueueIndex].ring[ringIdx].fromext);
+
+        vkr = ObjDisp(q)->QueueSubmit(Unwrap(q), 1, &submitInfo, VK_NULL_HANDLE);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+        // next submit needs to wait on fromext
+        unwrappedSems.assign(submitInfo.pSignalSemaphores, 1);
+        waitStage.resize(1);
+
+        submitInfo.pWaitDstStageMask = waitStage.data();
+        submitInfo.pWaitSemaphores = unwrappedSems.data();
+        submitInfo.waitSemaphoreCount = (uint32_t)unwrappedSems.size();
+
+        // and signal toext
+        submitInfo.pSignalSemaphores =
+            UnwrapPtr(m_ExternalQueues[swapQueueIndex].ring[ringIdx].toext);
       }
 
       int flags = activeWindow ? RenderDoc::eOverlay_ActiveWindow : 0;
@@ -839,15 +908,20 @@ VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR
 
       DoPipelineBarrier(textstate.cmd, 1, &bbBarrier);
 
-      ObjDisp(textstate.cmd)->EndCommandBuffer(Unwrap(textstate.cmd));
+      ObjDisp(cmd)->EndCommandBuffer(Unwrap(cmd));
 
-      rdcarray<VkPipelineStageFlags> waitStage;
-      waitStage.fill(unwrappedSems.size(), VK_PIPELINE_STAGE_ALL_COMMANDS_BIT);
-      SubmitCmds(unwrappedSems.data(), waitStage.data(), (uint32_t)unwrappedSems.size());
+      {
+        submitInfo.commandBufferCount = 1;
+        submitInfo.pCommandBuffers = UnwrapPtr(cmd);
+
+        vkr = ObjDisp(m_Queue)->QueueSubmit(Unwrap(m_Queue), 1, &submitInfo, VK_NULL_HANDLE);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
+      }
 
       if(swapQueueIndex != m_QueueFamilyIdx)
       {
-        VkCommandBuffer extQCmd = GetExtQueueCmd(swapQueueIndex);
+        VkCommandBuffer extQCmd = m_ExternalQueues[swapQueueIndex].ring[ringIdx].release;
+        VkFence fence = m_ExternalQueues[swapQueueIndex].ring[ringIdx].fence;
 
         vkr = vt->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
         RDCASSERTEQUAL(vkr, VK_SUCCESS);
@@ -856,10 +930,33 @@ VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR
 
         ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
 
-        SubmitAndFlushExtQueue(swapQueueIndex);
+        VkQueue q = m_ExternalQueues[swapQueueIndex].queue;
+
+        // wait on toext which was signalled above
+        unwrappedSems.assign(submitInfo.pSignalSemaphores, 1);
+        waitStage.resize(1);
+
+        submitInfo.pWaitDstStageMask = waitStage.data();
+        submitInfo.pWaitSemaphores = unwrappedSems.data();
+        submitInfo.waitSemaphoreCount = (uint32_t)unwrappedSems.size();
+
+        // release to the external queue
+        submitInfo.commandBufferCount = 1;
+        submitInfo.pCommandBuffers = UnwrapPtr(extQCmd);
+
+        // signal the overlaydone semaphore
+        submitInfo.signalSemaphoreCount = 1;
+        submitInfo.pSignalSemaphores = UnwrapPtr(sem);
+
+        // submit and signal the fence when we're done, so we know next time around that this is
+        // safe to re-use
+        vkr = ObjDisp(q)->QueueSubmit(Unwrap(q), 1, &submitInfo, Unwrap(fence));
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
       }
 
-      FlushQ();
+      // the present waits on our new semaphore
+      unwrappedInfo.waitSemaphoreCount = 1;
+      unwrappedInfo.pWaitSemaphores = submitInfo.pSignalSemaphores;
     }
 
     GetResourceManager()->CleanBackgroundFrameReferences();