Fix some cases missing multiqueue handling

renderdoc/driver/vulkan/vk_core.cpp

@@ -1297,6 +1297,8 @@ bool WrappedVulkan::EndFrameCapture(void *dev, void *wnd)
         {imInfo.extent.width, imInfo.extent.height, 1},
     };
 
+    uint32_t swapQueueIndex = m_ImageLayouts[GetResID(backbuffer)].queueFamilyIndex;
+
     VkImageMemoryBarrier bbBarrier = {
         VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
         NULL,
@@ -1304,31 +1306,49 @@ bool WrappedVulkan::EndFrameCapture(void *dev, void *wnd)
         VK_ACCESS_TRANSFER_READ_BIT,
         VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,
         VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
-        0,
-        0,    // MULTIDEVICE - need to actually pick the right queue family here maybe?
+        swapQueueIndex,
+        m_QueueFamilyIdx,
         Unwrap(backbuffer),
-        {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}};
+        {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1},
+    };
 
-    VkImageMemoryBarrier readBarrier = {VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
-                                        NULL,
-                                        0,
-                                        VK_ACCESS_TRANSFER_WRITE_BIT,
-                                        VK_IMAGE_LAYOUT_UNDEFINED,
-                                        VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
-                                        VK_QUEUE_FAMILY_IGNORED,
-                                        VK_QUEUE_FAMILY_IGNORED,
-                                        Unwrap(readbackIm),
-                                        {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}};
+    VkImageMemoryBarrier readBarrier = {
+        VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
+        NULL,
+        0,
+        VK_ACCESS_TRANSFER_WRITE_BIT,
+        VK_IMAGE_LAYOUT_UNDEFINED,
+        VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
+        VK_QUEUE_FAMILY_IGNORED,
+        VK_QUEUE_FAMILY_IGNORED,
+        Unwrap(readbackIm),
+        {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1},
+    };
 
     DoPipelineBarrier(cmd, 1, &bbBarrier);
     DoPipelineBarrier(cmd, 1, &readBarrier);
 
+    if(swapQueueIndex != m_QueueFamilyIdx)
+    {
+      VkCommandBuffer extQCmd = GetExtQueueCmd(swapQueueIndex);
+
+      vkr = vt->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+      DoPipelineBarrier(extQCmd, 1, &bbBarrier);
+
+      ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+
+      SubmitAndFlushExtQueue(swapQueueIndex);
+    }
+
     vt->CmdCopyImage(Unwrap(cmd), Unwrap(backbuffer), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
                      Unwrap(readbackIm), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &cpy);
 
     // barrier to switch backbuffer back to present layout
     std::swap(bbBarrier.oldLayout, bbBarrier.newLayout);
     std::swap(bbBarrier.srcAccessMask, bbBarrier.dstAccessMask);
+    std::swap(bbBarrier.srcQueueFamilyIndex, bbBarrier.dstQueueFamilyIndex);
 
     readBarrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
     readBarrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT;
@@ -1344,6 +1364,20 @@ bool WrappedVulkan::EndFrameCapture(void *dev, void *wnd)
     SubmitCmds();
     FlushQ();    // need to wait so we can readback
 
+    if(swapQueueIndex != m_QueueFamilyIdx)
+    {
+      VkCommandBuffer extQCmd = GetExtQueueCmd(swapQueueIndex);
+
+      vkr = vt->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+      DoPipelineBarrier(extQCmd, 1, &bbBarrier);
+
+      ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+
+      SubmitAndFlushExtQueue(swapQueueIndex);
+    }
+
     // map memory and readback
     byte *pData = NULL;
     vkr = vt->MapMemory(Unwrap(device), Unwrap(readbackMem.mem), readbackMem.offs, readbackMem.size,

renderdoc/driver/vulkan/vk_core.h

@@ -814,7 +814,7 @@ public:
   void Apply_InitialState(WrappedVkRes *live, VkInitialContents initial);
 
   void RemapQueueFamilyIndices(uint32_t &srcQueueFamily, uint32_t &dstQueueFamily);
-
+  uint32_t GetQueueFamilyIndex() { return m_QueueFamilyIdx; }
   bool ReleaseResource(WrappedVkRes *res);
 
   ReplayStatus Initialise(VkInitParams &params, uint64_t sectionVersion);

renderdoc/driver/vulkan/vk_debug.cpp

@@ -676,10 +676,11 @@ void VulkanDebugManager::CreateCustomShaderTex(uint32_t width, uint32_t height,
       VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,
       VK_IMAGE_LAYOUT_UNDEFINED,
       VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
-      0,
-      0,    // MULTIDEVICE - need to actually pick the right queue family here maybe?
+      VK_QUEUE_FAMILY_IGNORED,
+      VK_QUEUE_FAMILY_IGNORED,
       Unwrap(m_Custom.TexImg),
-      {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, 1}};
+      {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, 1},
+  };
 
   m_pDriver->m_ImageLayouts[GetResID(m_Custom.TexImg)].subresourceStates[0].newLayout =
       VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
@@ -1735,10 +1736,11 @@ void VulkanReplay::TextureRendering::Init(WrappedVulkan *driver, VkDescriptorPoo
             VK_ACCESS_SHADER_READ_BIT,
             VK_IMAGE_LAYOUT_UNDEFINED,
             VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
-            0,
-            0,    // MULTIDEVICE - need to actually pick the right queue family here maybe?
+            VK_QUEUE_FAMILY_IGNORED,
+            VK_QUEUE_FAMILY_IGNORED,
             Unwrap(DummyImages[index]),
-            {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}};
+            {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1},
+        };
 
         DoPipelineBarrier(cmd, 1, &barrier);
 
@@ -2213,10 +2215,11 @@ void VulkanReplay::PixelPicking::Init(WrappedVulkan *driver, VkDescriptorPool de
       VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,
       VK_IMAGE_LAYOUT_UNDEFINED,
       VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
-      0,
-      0,    // MULTIDEVICE - need to actually pick the right queue family here maybe?
+      VK_QUEUE_FAMILY_IGNORED,
+      VK_QUEUE_FAMILY_IGNORED,
       Unwrap(Image),
-      {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}};
+      {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1},
+  };
 
   DoPipelineBarrier(cmd, 1, &barrier);
 

renderdoc/driver/vulkan/vk_initstate.cpp

@@ -1289,8 +1289,8 @@ void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, VkInitialContents ini
             0,
             VK_IMAGE_LAYOUT_UNDEFINED,
             VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
-            VK_QUEUE_FAMILY_IGNORED,
-            VK_QUEUE_FAMILY_IGNORED,
+            m_ImageLayouts[id].queueFamilyIndex,
+            m_QueueFamilyIdx,
             ToHandle<VkImage>(live),
             {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS},
         };
@@ -1300,11 +1300,32 @@ void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, VkInitialContents ini
         // clear completes before subsequent operations
         barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
 
+        VkCommandBuffer extQCmd = VK_NULL_HANDLE;
+
+        if(barrier.srcQueueFamilyIndex != barrier.dstQueueFamilyIndex)
+        {
+          extQCmd = GetExtQueueCmd(barrier.srcQueueFamilyIndex);
+
+          vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+          RDCASSERTEQUAL(vkr, VK_SUCCESS);
+        }
+
         for(size_t si = 0; si < m_ImageLayouts[id].subresourceStates.size(); si++)
         {
           barrier.subresourceRange = m_ImageLayouts[id].subresourceStates[si].subresourceRange;
           barrier.oldLayout = m_ImageLayouts[id].subresourceStates[si].newLayout;
           DoPipelineBarrier(cmd, 1, &barrier);
+
+          if(extQCmd != VK_NULL_HANDLE)
+            DoPipelineBarrier(extQCmd, 1, &barrier);
+        }
+
+        if(extQCmd != VK_NULL_HANDLE)
+        {
+          vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+          RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+          SubmitAndFlushExtQueue(barrier.srcQueueFamilyIndex);
         }
 
         VkClearColorValue clearval = {};
@@ -1320,17 +1341,40 @@ void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, VkInitialContents ini
         barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
         barrier.dstAccessMask = VK_ACCESS_ALL_READ_BITS;
 
+        std::swap(barrier.srcQueueFamilyIndex, barrier.dstQueueFamilyIndex);
+
+        if(extQCmd != VK_NULL_HANDLE)
+        {
+          vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+          RDCASSERTEQUAL(vkr, VK_SUCCESS);
+        }
+
         for(size_t si = 0; si < m_ImageLayouts[id].subresourceStates.size(); si++)
         {
           barrier.subresourceRange = m_ImageLayouts[id].subresourceStates[si].subresourceRange;
           barrier.newLayout = m_ImageLayouts[id].subresourceStates[si].newLayout;
           barrier.dstAccessMask |= MakeAccessMask(barrier.newLayout);
           DoPipelineBarrier(cmd, 1, &barrier);
+
+          if(extQCmd != VK_NULL_HANDLE)
+            DoPipelineBarrier(extQCmd, 1, &barrier);
         }
 
         vkr = ObjDisp(cmd)->EndCommandBuffer(Unwrap(cmd));
         RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
+        if(extQCmd != VK_NULL_HANDLE)
+        {
+          // ensure work is completed before we pass ownership back to original queue
+          SubmitCmds();
+          FlushQ();
+
+          vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+          RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+          SubmitAndFlushExtQueue(barrier.dstQueueFamilyIndex);
+        }
+
 #if ENABLED(SINGLE_FLUSH_VALIDATE)
         SubmitCmds();
 #endif
@@ -1349,8 +1393,8 @@ void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, VkInitialContents ini
             0,
             VK_IMAGE_LAYOUT_UNDEFINED,
             VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
-            VK_QUEUE_FAMILY_IGNORED,
-            VK_QUEUE_FAMILY_IGNORED,
+            m_ImageLayouts[id].queueFamilyIndex,
+            m_QueueFamilyIdx,
             ToHandle<VkImage>(live),
             {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS},
         };
@@ -1360,11 +1404,32 @@ void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, VkInitialContents ini
         // clear completes before subsequent operations
         barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
 
+        VkCommandBuffer extQCmd = VK_NULL_HANDLE;
+
+        if(barrier.srcQueueFamilyIndex != barrier.dstQueueFamilyIndex)
+        {
+          extQCmd = GetExtQueueCmd(barrier.srcQueueFamilyIndex);
+
+          vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+          RDCASSERTEQUAL(vkr, VK_SUCCESS);
+        }
+
         for(size_t si = 0; si < m_ImageLayouts[id].subresourceStates.size(); si++)
         {
           barrier.subresourceRange = m_ImageLayouts[id].subresourceStates[si].subresourceRange;
           barrier.oldLayout = m_ImageLayouts[id].subresourceStates[si].newLayout;
           DoPipelineBarrier(cmd, 1, &barrier);
+
+          if(extQCmd != VK_NULL_HANDLE)
+            DoPipelineBarrier(extQCmd, 1, &barrier);
+        }
+
+        if(extQCmd != VK_NULL_HANDLE)
+        {
+          vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+          RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+          SubmitAndFlushExtQueue(barrier.srcQueueFamilyIndex);
         }
 
         VkClearDepthStencilValue clearval = {1.0f, 0};
@@ -1381,16 +1446,39 @@ void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, VkInitialContents ini
         barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
         barrier.dstAccessMask = VK_ACCESS_ALL_READ_BITS;
 
+        std::swap(barrier.srcQueueFamilyIndex, barrier.dstQueueFamilyIndex);
+
+        if(extQCmd != VK_NULL_HANDLE)
+        {
+          vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+          RDCASSERTEQUAL(vkr, VK_SUCCESS);
+        }
+
         for(size_t si = 0; si < m_ImageLayouts[id].subresourceStates.size(); si++)
         {
           barrier.subresourceRange = m_ImageLayouts[id].subresourceStates[si].subresourceRange;
           barrier.newLayout = m_ImageLayouts[id].subresourceStates[si].newLayout;
           DoPipelineBarrier(cmd, 1, &barrier);
+
+          if(extQCmd != VK_NULL_HANDLE)
+            DoPipelineBarrier(extQCmd, 1, &barrier);
         }
 
         vkr = ObjDisp(cmd)->EndCommandBuffer(Unwrap(cmd));
         RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
+        if(extQCmd != VK_NULL_HANDLE)
+        {
+          // ensure work is completed before we pass ownership back to original queue
+          SubmitCmds();
+          FlushQ();
+
+          vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+          RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+          SubmitAndFlushExtQueue(barrier.dstQueueFamilyIndex);
+        }
+
 #if ENABLED(SINGLE_FLUSH_VALIDATE)
         SubmitCmds();
 #endif
@@ -1430,8 +1518,8 @@ void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, VkInitialContents ini
           0,
           VK_IMAGE_LAYOUT_UNDEFINED,
           VK_IMAGE_LAYOUT_GENERAL,
-          VK_QUEUE_FAMILY_IGNORED,
-          VK_QUEUE_FAMILY_IGNORED,
+          m_ImageLayouts[id].queueFamilyIndex,
+          m_QueueFamilyIdx,
           ToHandle<VkImage>(live),
           {aspectFlags, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS},
       };
@@ -1440,11 +1528,32 @@ void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, VkInitialContents ini
       barrier.dstAccessMask =
           VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
 
+      VkCommandBuffer extQCmd = VK_NULL_HANDLE;
+
+      if(barrier.srcQueueFamilyIndex != barrier.dstQueueFamilyIndex)
+      {
+        extQCmd = GetExtQueueCmd(barrier.srcQueueFamilyIndex);
+
+        vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
+      }
+
       for(size_t si = 0; si < m_ImageLayouts[id].subresourceStates.size(); si++)
       {
         barrier.subresourceRange = m_ImageLayouts[id].subresourceStates[si].subresourceRange;
         barrier.oldLayout = m_ImageLayouts[id].subresourceStates[si].newLayout;
         DoPipelineBarrier(cmd, 1, &barrier);
+
+        if(extQCmd != VK_NULL_HANDLE)
+          DoPipelineBarrier(extQCmd, 1, &barrier);
+      }
+
+      if(extQCmd != VK_NULL_HANDLE)
+      {
+        vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+        SubmitAndFlushExtQueue(barrier.srcQueueFamilyIndex);
       }
 
       VkImage arrayIm = initial.img;
@@ -1467,17 +1576,40 @@ void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, VkInitialContents ini
           VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
       barrier.dstAccessMask = VK_ACCESS_ALL_READ_BITS;
 
+      std::swap(barrier.srcQueueFamilyIndex, barrier.dstQueueFamilyIndex);
+
+      if(extQCmd != VK_NULL_HANDLE)
+      {
+        vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
+      }
+
       for(size_t si = 0; si < m_ImageLayouts[id].subresourceStates.size(); si++)
       {
         barrier.subresourceRange = m_ImageLayouts[id].subresourceStates[si].subresourceRange;
         barrier.newLayout = m_ImageLayouts[id].subresourceStates[si].newLayout;
         barrier.dstAccessMask |= MakeAccessMask(barrier.newLayout);
         DoPipelineBarrier(cmd, 1, &barrier);
+
+        if(extQCmd != VK_NULL_HANDLE)
+          DoPipelineBarrier(extQCmd, 1, &barrier);
       }
 
       vkr = ObjDisp(cmd)->EndCommandBuffer(Unwrap(cmd));
       RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
+      if(extQCmd != VK_NULL_HANDLE)
+      {
+        // ensure work is completed before we pass ownership back to original queue
+        SubmitCmds();
+        FlushQ();
+
+        vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+        RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+        SubmitAndFlushExtQueue(barrier.dstQueueFamilyIndex);
+      }
+
 #if ENABLED(SINGLE_FLUSH_VALIDATE)
       SubmitCmds();
 #endif
@@ -1508,10 +1640,11 @@ void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, VkInitialContents ini
         0,
         VK_IMAGE_LAYOUT_UNDEFINED,
         VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
-        VK_QUEUE_FAMILY_IGNORED,
-        VK_QUEUE_FAMILY_IGNORED,
+        m_ImageLayouts[id].queueFamilyIndex,
+        m_QueueFamilyIdx,
         ToHandle<VkImage>(live),
-        {aspectFlags, 0, 1, 0, (uint32_t)m_CreationInfo.m_Image[id].arrayLayers}};
+        {aspectFlags, 0, 1, 0, (uint32_t)m_CreationInfo.m_Image[id].arrayLayers},
+    };
 
     if(aspectFlags == VK_IMAGE_ASPECT_DEPTH_BIT && !IsDepthOnlyFormat(fmt))
       dstimBarrier.subresourceRange.aspectMask |= VK_IMAGE_ASPECT_STENCIL_BIT;
@@ -1528,12 +1661,33 @@ void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, VkInitialContents ini
     dstimBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
     dstimBarrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
 
+    VkCommandBuffer extQCmd = VK_NULL_HANDLE;
+
+    if(dstimBarrier.srcQueueFamilyIndex != dstimBarrier.dstQueueFamilyIndex)
+    {
+      extQCmd = GetExtQueueCmd(dstimBarrier.srcQueueFamilyIndex);
+
+      vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+    }
+
     for(size_t si = 0; si < m_ImageLayouts[id].subresourceStates.size(); si++)
     {
       dstimBarrier.subresourceRange = m_ImageLayouts[id].subresourceStates[si].subresourceRange;
       dstimBarrier.oldLayout = m_ImageLayouts[id].subresourceStates[si].newLayout;
       dstimBarrier.srcAccessMask = VK_ACCESS_ALL_WRITE_BITS | MakeAccessMask(dstimBarrier.oldLayout);
       DoPipelineBarrier(cmd, 1, &dstimBarrier);
+
+      if(extQCmd != VK_NULL_HANDLE)
+        DoPipelineBarrier(extQCmd, 1, &dstimBarrier);
+    }
+
+    if(extQCmd != VK_NULL_HANDLE)
+    {
+      vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+      SubmitAndFlushExtQueue(dstimBarrier.srcQueueFamilyIndex);
     }
 
     // copy each slice/mip individually
@@ -1594,17 +1748,40 @@ void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, VkInitialContents ini
     dstimBarrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
     dstimBarrier.dstAccessMask = VK_ACCESS_ALL_READ_BITS;
 
+    std::swap(dstimBarrier.srcQueueFamilyIndex, dstimBarrier.dstQueueFamilyIndex);
+
+    if(extQCmd != VK_NULL_HANDLE)
+    {
+      vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+    }
+
     for(size_t si = 0; si < m_ImageLayouts[id].subresourceStates.size(); si++)
     {
       dstimBarrier.subresourceRange = m_ImageLayouts[id].subresourceStates[si].subresourceRange;
       dstimBarrier.newLayout = m_ImageLayouts[id].subresourceStates[si].newLayout;
       dstimBarrier.dstAccessMask |= MakeAccessMask(dstimBarrier.newLayout);
       DoPipelineBarrier(cmd, 1, &dstimBarrier);
+
+      if(extQCmd != VK_NULL_HANDLE)
+        DoPipelineBarrier(extQCmd, 1, &dstimBarrier);
     }
 
     vkr = ObjDisp(cmd)->EndCommandBuffer(Unwrap(cmd));
     RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
+    if(extQCmd != VK_NULL_HANDLE)
+    {
+      // ensure work is completed before we pass ownership back to original queue
+      SubmitCmds();
+      FlushQ();
+
+      vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+      SubmitAndFlushExtQueue(dstimBarrier.dstQueueFamilyIndex);
+    }
+
 #if ENABLED(SINGLE_FLUSH_VALIDATE)
     SubmitCmds();
 #endif

renderdoc/driver/vulkan/vk_manager.cpp

@@ -286,6 +286,7 @@ void VulkanResourceManager::SerialiseImageStates(SerialiserType &ser,
         t.srcQueueFamilyIndex = ImageState.queueFamilyIndex;
         t.dstQueueFamilyIndex = ImageState.queueFamilyIndex;
         m_Core->RemapQueueFamilyIndices(t.srcQueueFamilyIndex, t.dstQueueFamilyIndex);
+        state.dstQueueFamilyIndex = t.dstQueueFamilyIndex;
         t.image = Unwrap(GetCurrentHandle<VkImage>(liveid));
         t.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
         ReplacePresentableImageLayout(state.newLayout);

renderdoc/driver/vulkan/vk_outputwindow.cpp

@@ -69,10 +69,11 @@ VulkanReplay::OutputWindow::OutputWindow()
       0,
       VK_IMAGE_LAYOUT_UNDEFINED,
       VK_IMAGE_LAYOUT_UNDEFINED,
-      0,
-      0,    // MULTIDEVICE - need to actually pick the right queue family here maybe?
+      VK_QUEUE_FAMILY_IGNORED,
+      VK_QUEUE_FAMILY_IGNORED,
       VK_NULL_HANDLE,
-      {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}};
+      {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1},
+  };
   for(size_t i = 0; i < ARRAY_COUNT(colBarrier); i++)
     colBarrier[i] = t;
 

renderdoc/driver/vulkan/vk_overlay.cpp

@@ -513,25 +513,28 @@ ResourceId VulkanReplay::RenderOverlay(ResourceId texid, CompType typeHint, Debu
         VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,
         VK_IMAGE_LAYOUT_UNDEFINED,
         VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
-        0,
-        0,    // MULTIDEVICE - need to actually pick the right queue family here maybe?
+        VK_QUEUE_FAMILY_IGNORED,
+        VK_QUEUE_FAMILY_IGNORED,
         Unwrap(m_Overlay.Image),
-        {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}};
+        {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1},
+    };
 
     m_pDriver->m_ImageLayouts[GetResID(m_Overlay.Image)].subresourceStates[0].newLayout =
         VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
 
     DoPipelineBarrier(cmd, 1, &barrier);
 
-    VkAttachmentDescription colDesc = {0,
-                                       imInfo.format,
-                                       imInfo.samples,
-                                       VK_ATTACHMENT_LOAD_OP_LOAD,
-                                       VK_ATTACHMENT_STORE_OP_STORE,
-                                       VK_ATTACHMENT_LOAD_OP_DONT_CARE,
-                                       VK_ATTACHMENT_STORE_OP_DONT_CARE,
-                                       VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
-                                       VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL};
+    VkAttachmentDescription colDesc = {
+        0,
+        imInfo.format,
+        imInfo.samples,
+        VK_ATTACHMENT_LOAD_OP_LOAD,
+        VK_ATTACHMENT_STORE_OP_STORE,
+        VK_ATTACHMENT_LOAD_OP_DONT_CARE,
+        VK_ATTACHMENT_STORE_OP_DONT_CARE,
+        VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
+        VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
+    };
 
     VkAttachmentReference colRef = {0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL};
 
@@ -1697,10 +1700,11 @@ ResourceId VulkanReplay::RenderOverlay(ResourceId texid, CompType typeHint, Debu
           VK_ACCESS_TRANSFER_WRITE_BIT,
           VK_IMAGE_LAYOUT_UNDEFINED,
           VK_IMAGE_LAYOUT_GENERAL,
-          0,
-          0,    // MULTIDEVICE - need to actually pick the right queue family here maybe?
+          VK_QUEUE_FAMILY_IGNORED,
+          VK_QUEUE_FAMILY_IGNORED,
           Unwrap(quadImg),
-          {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 4}};
+          {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 4},
+      };
 
       // clear all to black
       DoPipelineBarrier(cmd, 1, &quadImBarrier);

renderdoc/driver/vulkan/vk_rendertext.cpp

@@ -406,8 +406,8 @@ VulkanTextRenderer::VulkanTextRenderer(WrappedVulkan *driver)
         VK_ACCESS_HOST_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT,
         VK_IMAGE_LAYOUT_UNDEFINED,
         VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
-        0,
-        0,    // MULTIDEVICE - need to actually pick the right queue family here maybe?
+        VK_QUEUE_FAMILY_IGNORED,
+        VK_QUEUE_FAMILY_IGNORED,
         Unwrap(m_TextAtlas),
         {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1},
     };
@@ -453,8 +453,8 @@ VulkanTextRenderer::VulkanTextRenderer(WrappedVulkan *driver)
         VK_ACCESS_SHADER_READ_BIT,
         VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
         VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
-        0,
-        0,    // MULTIDEVICE - need to actually pick the right queue family here maybe?
+        VK_QUEUE_FAMILY_IGNORED,
+        VK_QUEUE_FAMILY_IGNORED,
         Unwrap(m_TextAtlas),
         {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1},
     };

renderdoc/driver/vulkan/vk_rendertexture.cpp

@@ -371,8 +371,8 @@ bool VulkanReplay::RenderTextureInternal(TextureDisplay cfg, VkRenderPassBeginIn
       0,
       VK_IMAGE_LAYOUT_UNDEFINED,
       VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
-      VK_QUEUE_FAMILY_IGNORED,
-      VK_QUEUE_FAMILY_IGNORED,
+      layouts.queueFamilyIndex,
+      m_pDriver->GetQueueFamilyIndex(),
       Unwrap(liveIm),
       {0, 0, 1, 0, 1}    // will be overwritten by subresourceRange
   };
@@ -387,12 +387,34 @@ bool VulkanReplay::RenderTextureInternal(TextureDisplay cfg, VkRenderPassBeginIn
 
   vt->BeginCommandBuffer(Unwrap(cmd), &beginInfo);
 
+  VkCommandBuffer extQCmd = VK_NULL_HANDLE;
+  VkResult vkr = VK_SUCCESS;
+
+  if(srcimBarrier.srcQueueFamilyIndex != srcimBarrier.dstQueueFamilyIndex)
+  {
+    extQCmd = m_pDriver->GetExtQueueCmd(srcimBarrier.srcQueueFamilyIndex);
+
+    vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+    RDCASSERTEQUAL(vkr, VK_SUCCESS);
+  }
+
   for(size_t si = 0; si < layouts.subresourceStates.size(); si++)
   {
     srcimBarrier.subresourceRange = layouts.subresourceStates[si].subresourceRange;
     srcimBarrier.oldLayout = layouts.subresourceStates[si].newLayout;
     srcimBarrier.srcAccessMask = VK_ACCESS_ALL_WRITE_BITS | MakeAccessMask(srcimBarrier.oldLayout);
     DoPipelineBarrier(cmd, 1, &srcimBarrier);
+
+    if(extQCmd != VK_NULL_HANDLE)
+      DoPipelineBarrier(extQCmd, 1, &srcimBarrier);
+  }
+
+  if(extQCmd != VK_NULL_HANDLE)
+  {
+    vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+    RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+    m_pDriver->SubmitAndFlushExtQueue(layouts.queueFamilyIndex);
   }
 
   srcimBarrier.oldLayout = srcimBarrier.newLayout;
@@ -446,16 +468,39 @@ bool VulkanReplay::RenderTextureInternal(TextureDisplay cfg, VkRenderPassBeginIn
     vt->CmdEndRenderPass(Unwrap(cmd));
   }
 
+  std::swap(srcimBarrier.srcQueueFamilyIndex, srcimBarrier.dstQueueFamilyIndex);
+
+  if(extQCmd != VK_NULL_HANDLE)
+  {
+    vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+    RDCASSERTEQUAL(vkr, VK_SUCCESS);
+  }
+
   for(size_t si = 0; si < layouts.subresourceStates.size(); si++)
   {
     srcimBarrier.subresourceRange = layouts.subresourceStates[si].subresourceRange;
     srcimBarrier.newLayout = layouts.subresourceStates[si].newLayout;
     srcimBarrier.dstAccessMask = MakeAccessMask(srcimBarrier.newLayout);
     DoPipelineBarrier(cmd, 1, &srcimBarrier);
+
+    if(extQCmd != VK_NULL_HANDLE)
+      DoPipelineBarrier(extQCmd, 1, &srcimBarrier);
   }
 
   vt->EndCommandBuffer(Unwrap(cmd));
 
+  if(extQCmd != VK_NULL_HANDLE)
+  {
+    // ensure work is completed before we pass ownership back to original queue
+    m_pDriver->SubmitCmds();
+    m_pDriver->FlushQ();
+
+    vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+    RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+    m_pDriver->SubmitAndFlushExtQueue(layouts.queueFamilyIndex);
+  }
+
 #if ENABLED(SINGLE_FLUSH_VALIDATE)
   m_pDriver->SubmitCmds();
 #endif

renderdoc/driver/vulkan/vk_replay.cpp

@@ -2294,6 +2294,8 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
   VkImage tmpImage = VK_NULL_HANDLE;
   VkDeviceMemory tmpMemory = VK_NULL_HANDLE;
 
+  uint32_t srcQueueIndex = layouts.queueFamilyIndex;
+
   VkFramebuffer *tmpFB = NULL;
   VkImageView *tmpView = NULL;
   uint32_t numFBs = 0;
@@ -2318,6 +2320,8 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
     wasms = true;
   }
 
+  VkCommandBuffer extQCmd = VK_NULL_HANDLE;
+
   if(params.remap != RemapTexture::NoRemap)
   {
     int renderFlags = 0;
@@ -2386,10 +2390,11 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
         0,
         VK_IMAGE_LAYOUT_UNDEFINED,
         VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
-        0,
-        0,    // MULTIDEVICE - need to actually pick the right queue family here maybe?
+        VK_QUEUE_FAMILY_IGNORED,
+        VK_QUEUE_FAMILY_IGNORED,
         tmpImage,
-        {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}};
+        {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS},
+    };
 
     // move tmp image into transfer destination layout
     DoPipelineBarrier(cmd, 1, &dstimBarrier);
@@ -2527,6 +2532,7 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
     m_DebugHeight = oldH;
 
     srcImage = tmpImage;
+    srcQueueIndex = m_pDriver->GetQueueFamilyIndex();
 
     // fetch a new command buffer for copy & readback
     cmd = m_pDriver->GetNextCmd();
@@ -2594,10 +2600,11 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
         0,
         VK_IMAGE_LAYOUT_UNDEFINED,
         VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
-        VK_QUEUE_FAMILY_IGNORED,
-        VK_QUEUE_FAMILY_IGNORED,
+        srcQueueIndex,
+        m_pDriver->GetQueueFamilyIndex(),
         srcImage,
-        {srcAspectMask, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}};
+        {srcAspectMask, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS},
+    };
 
     VkImageMemoryBarrier dstimBarrier = {
         VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
@@ -2606,21 +2613,41 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
         0,
         VK_IMAGE_LAYOUT_UNDEFINED,
         VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
-        0,
-        0,    // MULTIDEVICE - need to actually pick the right queue family here maybe?
+        VK_QUEUE_FAMILY_IGNORED,
+        VK_QUEUE_FAMILY_IGNORED,
         tmpImage,
-        {srcAspectMask, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}};
+        {srcAspectMask, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS},
+    };
 
     // ensure all previous writes have completed
     srcimBarrier.srcAccessMask = VK_ACCESS_ALL_WRITE_BITS;
     // before we go resolving
     srcimBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
 
+    if(srcimBarrier.srcQueueFamilyIndex != srcimBarrier.dstQueueFamilyIndex)
+    {
+      extQCmd = m_pDriver->GetExtQueueCmd(srcimBarrier.srcQueueFamilyIndex);
+
+      vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+    }
+
     for(size_t si = 0; si < layouts.subresourceStates.size(); si++)
     {
       srcimBarrier.subresourceRange = layouts.subresourceStates[si].subresourceRange;
       srcimBarrier.oldLayout = layouts.subresourceStates[si].newLayout;
       DoPipelineBarrier(cmd, 1, &srcimBarrier);
+
+      if(extQCmd != VK_NULL_HANDLE)
+        DoPipelineBarrier(extQCmd, 1, &srcimBarrier);
+    }
+
+    if(extQCmd != VK_NULL_HANDLE)
+    {
+      vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+      m_pDriver->SubmitAndFlushExtQueue(layouts.queueFamilyIndex);
     }
 
     srcimBarrier.oldLayout = srcimBarrier.newLayout;
@@ -2635,12 +2662,23 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
     vt->CmdResolveImage(Unwrap(cmd), srcImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
                         Unwrap(tmpImage), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &resolveRegion);
 
+    std::swap(srcimBarrier.srcQueueFamilyIndex, srcimBarrier.dstQueueFamilyIndex);
+
+    if(extQCmd != VK_NULL_HANDLE)
+    {
+      vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+    }
+
     // image layout back to normal
     for(size_t si = 0; si < layouts.subresourceStates.size(); si++)
     {
       srcimBarrier.subresourceRange = layouts.subresourceStates[si].subresourceRange;
       srcimBarrier.newLayout = layouts.subresourceStates[si].newLayout;
       DoPipelineBarrier(cmd, 1, &srcimBarrier);
+
+      if(extQCmd != VK_NULL_HANDLE)
+        DoPipelineBarrier(extQCmd, 1, &srcimBarrier);
     }
 
     // wait for resolve to finish before copy to buffer
@@ -2651,7 +2689,31 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
 
     DoPipelineBarrier(cmd, 1, &dstimBarrier);
 
+    if(extQCmd != VK_NULL_HANDLE)
+    {
+      // ensure this resolve happens before handing back the source image to the original queue
+      vkr = vt->EndCommandBuffer(Unwrap(cmd));
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+      m_pDriver->SubmitCmds();
+      m_pDriver->FlushQ();
+
+      vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+      m_pDriver->SubmitAndFlushExtQueue(layouts.queueFamilyIndex);
+
+      extQCmd = VK_NULL_HANDLE;
+
+      // fetch a new command buffer for remaining work
+      cmd = m_pDriver->GetNextCmd();
+
+      vkr = vt->BeginCommandBuffer(Unwrap(cmd), &beginInfo);
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+    }
+
     srcImage = tmpImage;
+    srcQueueIndex = m_pDriver->GetQueueFamilyIndex();
 
     // these have already been selected, don't need to fetch that subresource
     // when copying back to readback buffer
@@ -2697,10 +2759,11 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
         0,
         VK_IMAGE_LAYOUT_UNDEFINED,
         VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
-        VK_QUEUE_FAMILY_IGNORED,
-        VK_QUEUE_FAMILY_IGNORED,
+        srcQueueIndex,
+        m_pDriver->GetQueueFamilyIndex(),
         srcImage,
-        {srcAspectMask, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}};
+        {srcAspectMask, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS},
+    };
 
     VkImageMemoryBarrier dstimBarrier = {
         VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
@@ -2709,21 +2772,41 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
         0,
         VK_IMAGE_LAYOUT_UNDEFINED,
         VK_IMAGE_LAYOUT_GENERAL,
-        0,
-        0,    // MULTIDEVICE - need to actually pick the right queue family here maybe?
+        VK_QUEUE_FAMILY_IGNORED,
+        VK_QUEUE_FAMILY_IGNORED,
         tmpImage,
-        {srcAspectMask, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}};
+        {srcAspectMask, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS},
+    };
 
     // ensure all previous writes have completed
     srcimBarrier.srcAccessMask = VK_ACCESS_ALL_WRITE_BITS;
     // before we go copying to array
     srcimBarrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
 
+    if(srcimBarrier.srcQueueFamilyIndex != srcimBarrier.dstQueueFamilyIndex)
+    {
+      extQCmd = m_pDriver->GetExtQueueCmd(srcimBarrier.srcQueueFamilyIndex);
+
+      vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+    }
+
     for(size_t si = 0; si < layouts.subresourceStates.size(); si++)
     {
       srcimBarrier.subresourceRange = layouts.subresourceStates[si].subresourceRange;
       srcimBarrier.oldLayout = layouts.subresourceStates[si].newLayout;
       DoPipelineBarrier(cmd, 1, &srcimBarrier);
+
+      if(extQCmd != VK_NULL_HANDLE)
+        DoPipelineBarrier(extQCmd, 1, &srcimBarrier);
+    }
+
+    if(extQCmd != VK_NULL_HANDLE)
+    {
+      vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+      m_pDriver->SubmitAndFlushExtQueue(layouts.queueFamilyIndex);
     }
 
     srcimBarrier.oldLayout = srcimBarrier.newLayout;
@@ -2750,6 +2833,14 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
 
     srcimBarrier.srcAccessMask = VK_ACCESS_SHADER_READ_BIT;
 
+    if(extQCmd != VK_NULL_HANDLE)
+    {
+      vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+    }
+
+    std::swap(srcimBarrier.srcQueueFamilyIndex, srcimBarrier.dstQueueFamilyIndex);
+
     // image layout back to normal
     for(size_t si = 0; si < layouts.subresourceStates.size(); si++)
     {
@@ -2757,6 +2848,9 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
       srcimBarrier.newLayout = layouts.subresourceStates[si].newLayout;
       srcimBarrier.dstAccessMask = MakeAccessMask(srcimBarrier.newLayout);
       DoPipelineBarrier(cmd, 1, &srcimBarrier);
+
+      if(extQCmd != VK_NULL_HANDLE)
+        DoPipelineBarrier(extQCmd, 1, &srcimBarrier);
     }
 
     // wait for copy to finish before copy to buffer
@@ -2767,7 +2861,31 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
 
     DoPipelineBarrier(cmd, 1, &dstimBarrier);
 
+    if(extQCmd != VK_NULL_HANDLE)
+    {
+      // ensure this resolve happens before handing back the source image to the original queue
+      vkr = vt->EndCommandBuffer(Unwrap(cmd));
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+      m_pDriver->SubmitCmds();
+      m_pDriver->FlushQ();
+
+      vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+      m_pDriver->SubmitAndFlushExtQueue(layouts.queueFamilyIndex);
+
+      extQCmd = VK_NULL_HANDLE;
+
+      // fetch a new command buffer for remaining work
+      cmd = m_pDriver->GetNextCmd();
+
+      vkr = vt->BeginCommandBuffer(Unwrap(cmd), &beginInfo);
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+    }
+
     srcImage = tmpImage;
+    srcQueueIndex = m_pDriver->GetQueueFamilyIndex();
   }
 
   VkImageMemoryBarrier srcimBarrier = {
@@ -2777,10 +2895,11 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
       0,
       VK_IMAGE_LAYOUT_UNDEFINED,
       VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
-      VK_QUEUE_FAMILY_IGNORED,
-      VK_QUEUE_FAMILY_IGNORED,
+      srcQueueIndex,
+      m_pDriver->GetQueueFamilyIndex(),
       srcImage,
-      {srcAspectMask, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}};
+      {srcAspectMask, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS},
+  };
 
   // if we have no tmpImage, we're copying directly from the real image
   if(tmpImage == VK_NULL_HANDLE)
@@ -2790,11 +2909,30 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
     // before we go resolving
     srcimBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
 
+    if(srcimBarrier.srcQueueFamilyIndex != srcimBarrier.dstQueueFamilyIndex)
+    {
+      extQCmd = m_pDriver->GetExtQueueCmd(srcimBarrier.srcQueueFamilyIndex);
+
+      vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+    }
+
     for(size_t si = 0; si < layouts.subresourceStates.size(); si++)
     {
       srcimBarrier.subresourceRange = layouts.subresourceStates[si].subresourceRange;
       srcimBarrier.oldLayout = layouts.subresourceStates[si].newLayout;
       DoPipelineBarrier(cmd, 1, &srcimBarrier);
+
+      if(extQCmd != VK_NULL_HANDLE)
+        DoPipelineBarrier(extQCmd, 1, &srcimBarrier);
+    }
+
+    if(extQCmd != VK_NULL_HANDLE)
+    {
+      vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+      m_pDriver->SubmitAndFlushExtQueue(layouts.queueFamilyIndex);
     }
   }
 
@@ -2914,6 +3052,14 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
     // ensure transfer has completed
     srcimBarrier.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
 
+    std::swap(srcimBarrier.srcQueueFamilyIndex, srcimBarrier.dstQueueFamilyIndex);
+
+    if(extQCmd != VK_NULL_HANDLE)
+    {
+      vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo);
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+    }
+
     // image layout back to normal
     for(size_t si = 0; si < layouts.subresourceStates.size(); si++)
     {
@@ -2921,6 +3067,9 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
       srcimBarrier.newLayout = layouts.subresourceStates[si].newLayout;
       srcimBarrier.dstAccessMask = MakeAccessMask(srcimBarrier.newLayout);
       DoPipelineBarrier(cmd, 1, &srcimBarrier);
+
+      if(extQCmd != VK_NULL_HANDLE)
+        DoPipelineBarrier(extQCmd, 1, &srcimBarrier);
     }
   }
 
@@ -2944,6 +3093,16 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
   m_pDriver->SubmitCmds();
   m_pDriver->FlushQ();
 
+  if(extQCmd != VK_NULL_HANDLE)
+  {
+    vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd));
+    RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+    m_pDriver->SubmitAndFlushExtQueue(layouts.queueFamilyIndex);
+
+    extQCmd = VK_NULL_HANDLE;
+  }
+
   // map the buffer and copy to return buffer
   byte *pData = NULL;
   vkr = vt->MapMemory(Unwrap(dev), readbackMem, 0, VK_WHOLE_SIZE, 0, (void **)&pData);

renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp

@@ -567,10 +567,13 @@ void WrappedVulkan::Shutdown()
 
   for(size_t i = 0; i < m_ExternalQueues.size(); i++)
   {
-    GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].buffer);
+    if(m_ExternalQueues[i].buffer != VK_NULL_HANDLE)
+    {
+      GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].buffer);
 
-    ObjDisp(m_Device)->DestroyCommandPool(Unwrap(m_Device), Unwrap(m_ExternalQueues[i].pool), NULL);
-    GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].pool);
+      ObjDisp(m_Device)->DestroyCommandPool(Unwrap(m_Device), Unwrap(m_ExternalQueues[i].pool), NULL);
+      GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].pool);
+    }
   }
 
   FreeAllMemory(MemoryScope::InitialContents);
@@ -1540,6 +1543,38 @@ bool WrappedVulkan::Serialise_vkCreateDevice(SerialiserType &ser, VkPhysicalDevi
       GetResourceManager()->WrapResource(Unwrap(device), m_InternalCmds.cmdpool);
     }
 
+    // for each queue family that isn't our own, create a command pool and command buffer on that
+    // queue
+    for(uint32_t i = 0; i < createInfo.queueCreateInfoCount; i++)
+    {
+      uint32_t qidx = createInfo.pQueueCreateInfos[i].queueFamilyIndex;
+      m_ExternalQueues.resize(RDCMAX((uint32_t)m_ExternalQueues.size(), qidx + 1));
+
+      VkCommandPoolCreateInfo poolInfo = {
+          VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, NULL,
+          VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, qidx,
+      };
+      vkr = ObjDisp(device)->CreateCommandPool(Unwrap(device), &poolInfo, NULL,
+                                               &m_ExternalQueues[qidx].pool);
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+      GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].pool);
+
+      VkCommandBufferAllocateInfo cmdInfo = {
+          VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
+          NULL,
+          Unwrap(m_ExternalQueues[qidx].pool),
+          VK_COMMAND_BUFFER_LEVEL_PRIMARY,
+          1,
+      };
+
+      vkr = ObjDisp(device)->AllocateCommandBuffers(Unwrap(device), &cmdInfo,
+                                                    &m_ExternalQueues[qidx].buffer);
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+      GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].buffer);
+    }
+
     ObjDisp(physicalDevice)
         ->GetPhysicalDeviceProperties(Unwrap(physicalDevice), &m_PhysicalDeviceData.props);
 

renderdoc/driver/vulkan/wrappers/vk_queue_funcs.cpp

@@ -61,6 +61,16 @@ bool WrappedVulkan::Serialise_vkGetDeviceQueue(SerialiserType &ser, VkDevice dev
       SubmitCmds();
     }
 
+    if(remapFamily < m_ExternalQueues.size())
+    {
+      if(m_ExternalQueues[remapFamily].queue == VK_NULL_HANDLE)
+        m_ExternalQueues[remapFamily].queue = queue;
+    }
+    else
+    {
+      RDCERR("Unexpected queue family index %u", remapFamily);
+    }
+
     m_CreationInfo.m_Queue[GetResID(queue)] = remapFamily;
 
     AddResource(Queue, ResourceType::Queue, "Queue");
@@ -1266,6 +1276,16 @@ bool WrappedVulkan::Serialise_vkGetDeviceQueue2(SerialiserType &ser, VkDevice de
       SubmitCmds();
     }
 
+    if(remapFamily < m_ExternalQueues.size())
+    {
+      if(m_ExternalQueues[remapFamily].queue == VK_NULL_HANDLE)
+        m_ExternalQueues[remapFamily].queue = queue;
+    }
+    else
+    {
+      RDCERR("Unexpected queue family index %u", remapFamily);
+    }
+
     m_CreationInfo.m_Queue[GetResID(queue)] = remapFamily;
 
     AddResource(Queue, ResourceType::Queue, "Queue");