Fix backbuffer readbacks on headless mesh outputs on Vulkan

* We were trying to readback from an MSAA texture - not allowed. Need to resolve
  first.

renderdoc/driver/d3d11/d3d11_outputwindow.cpp

@@ -53,7 +53,7 @@ void D3D11Replay::OutputWindow::MakeRTV()
     texDesc.CPUAccessFlags = 0;
     texDesc.MipLevels = 1;
     texDesc.MiscFlags = 0;
-    texDesc.SampleDesc.Count = 1;
+    texDesc.SampleDesc.Count = multisampled ? 4 : 1;
     texDesc.SampleDesc.Quality = 0;
     texDesc.Usage = D3D11_USAGE_DEFAULT;
     texDesc.Width = width;
@@ -132,6 +132,7 @@ uint64_t D3D11Replay::MakeOutputWindow(WindowingData window, bool depth)
   DXGI_SWAP_CHAIN_DESC swapDesc = {};
   OutputWindow outw = {};
   outw.dev = m_pDevice;
+  outw.multisampled = depth;
 
   if(window.system == WindowingSystem::Win32)
   {
@@ -301,6 +302,8 @@ void D3D11Replay::GetOutputWindowData(uint64_t id, bytebuf &retData)
     return;
   }
 
+  texture->Release();
+
   ID3D11Texture2D *readback = NULL;
 
   D3D11_TEXTURE2D_DESC texDesc;
@@ -309,21 +312,47 @@ void D3D11Replay::GetOutputWindowData(uint64_t id, bytebuf &retData)
   texDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
   texDesc.BindFlags = 0;
   texDesc.Usage = D3D11_USAGE_STAGING;
+  texDesc.SampleDesc.Count = 1;
 
   HRESULT hr = m_pDevice->CreateTexture2D(&texDesc, NULL, &readback);
 
   if(FAILED(hr))
   {
     RDCERR("Couldn't create staging texture for readback, HRESULT: %s", ToStr(hr).c_str());
-    SAFE_RELEASE(texture);
+    SAFE_RELEASE(readback);
     return;
   }
 
+  ID3D11Texture2D *resolve = NULL;
+
+  if(outw.multisampled)
+  {
+    texDesc.CPUAccessFlags = 0;
+    texDesc.Usage = D3D11_USAGE_DEFAULT;
+
+    hr = m_pDevice->CreateTexture2D(&texDesc, NULL, &resolve);
+
+    if(FAILED(hr))
+    {
+      RDCERR("Couldn't create staging texture for readback, HRESULT: %s", ToStr(hr).c_str());
+      SAFE_RELEASE(readback);
+      SAFE_RELEASE(resolve);
+      return;
+    }
+  }
+
   ID3D11DeviceContext *ctx = m_pDevice->GetImmediateContext();
 
-  ctx->CopyResource(readback, texture);
-
-  SAFE_RELEASE(texture);
+  if(outw.multisampled)
+  {
+    ctx->ResolveSubresource(resolve, 0, texture, 0, texDesc.Format);
+    ctx->CopyResource(readback, resolve);
+    SAFE_RELEASE(resolve);
+  }
+  else
+  {
+    ctx->CopyResource(readback, texture);
+  }
 
   D3D11_MAPPED_SUBRESOURCE mapped = {};
   ctx->Map(readback, 0, D3D11_MAP_READ, 0, &mapped);

renderdoc/driver/d3d11/d3d11_replay.h

@@ -309,6 +309,7 @@ private:
     void MakeDSV();
 
     int width, height;
+    bool multisampled;
   };
 
   float m_OutputWidth = 1.0f;

renderdoc/driver/d3d12/d3d12_outputwindow.cpp

@@ -37,10 +37,6 @@ void D3D12Replay::OutputWindow::MakeRTV(bool msaa)
   if(bbDesc.Width)
   {
     texDesc = bbDesc;
-
-    texDesc.SampleDesc.Count = msaa ? D3D12_MSAA_SAMPLECOUNT : 1;
-
-    multisampled = msaa;
   }
   else
   {
@@ -51,10 +47,10 @@ void D3D12Replay::OutputWindow::MakeRTV(bool msaa)
     texDesc.MipLevels = 1;
     texDesc.SampleDesc.Count = 1;
     texDesc.SampleDesc.Quality = 0;
-
-    multisampled = false;
   }
 
+  texDesc.SampleDesc.Count = msaa ? D3D12_MSAA_SAMPLECOUNT : 1;
+
   texDesc.Height = height;
   texDesc.Width = width;
 
@@ -90,7 +86,7 @@ void D3D12Replay::OutputWindow::MakeRTV(bool msaa)
     texDesc.SampleDesc.Count = 1;
 
     hr = dev->CreateCommittedResource(&heapProps, D3D12_HEAP_FLAG_NONE, &texDesc,
-                                      D3D12_RESOURCE_STATE_RENDER_TARGET, NULL,
+                                      D3D12_RESOURCE_STATE_COPY_SOURCE, NULL,
                                       __uuidof(ID3D12Resource), (void **)&colResolve);
 
     col->SetName(L"Output Window Resolve");
@@ -449,6 +445,28 @@ void D3D12Replay::GetOutputWindowData(uint64_t id, bytebuf &retData)
     dst.pResource = readback;
     dst.PlacedFootprint = layout;
 
+    // resolve or copy from colour to backbuffer
+    if(outw.multisampled)
+    {
+      D3D12_RESOURCE_BARRIER resolvebarrier = {};
+
+      resolvebarrier.Transition.pResource = outw.colResolve;
+      resolvebarrier.Transition.StateBefore = D3D12_RESOURCE_STATE_COPY_SOURCE;
+      resolvebarrier.Transition.StateAfter = D3D12_RESOURCE_STATE_RESOLVE_DEST;
+
+      list->ResourceBarrier(1, &resolvebarrier);
+
+      // resolve then copy, as the resolve can't go from SRGB to non-SRGB target
+      list->ResolveSubresource(outw.colResolve, 0, outw.col, 0, DXGI_FORMAT_R8G8B8A8_UNORM_SRGB);
+
+      std::swap(resolvebarrier.Transition.StateBefore, resolvebarrier.Transition.StateAfter);
+
+      // now move the resolve target into copy source
+      list->ResourceBarrier(1, &resolvebarrier);
+
+      src.pResource = outw.colResolve;
+    }
+
     list->CopyTextureRegion(&dst, 0, 0, 0, &src, NULL);
 
     // transition back

renderdoc/driver/vulkan/vk_outputwindow.cpp

@@ -699,7 +699,45 @@ void VulkanReplay::GetOutputWindowData(uint64_t id, bytebuf &retData)
 
   DoPipelineBarrier(cmd, 1, &outw.bbBarrier);
 
-  vt->CmdCopyImageToBuffer(Unwrap(cmd), Unwrap(outw.bb), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
+  VkImage copySource = outw.bb;
+
+  if(outw.resolveimg != VK_NULL_HANDLE)
+  {
+    VkImageResolve resolve = {
+        {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, {0, 0, 0},
+        {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, {0, 0, 0},
+        {outw.width, outw.height, 1},
+    };
+
+    VkImageMemoryBarrier resolveBarrier = {
+        VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
+        NULL,
+        VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT,
+        VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT,
+        VK_IMAGE_LAYOUT_UNDEFINED,
+        VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
+        VK_QUEUE_FAMILY_IGNORED,
+        VK_QUEUE_FAMILY_IGNORED,
+        Unwrap(outw.resolveimg),
+        {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1},
+    };
+
+    // discard previous contents of resolve buffer and finish any work with it.
+    DoPipelineBarrier(cmd, 1, &resolveBarrier);
+
+    // resolve from the backbuffer to resolve buffer (identical format)
+    vt->CmdResolveImage(Unwrap(cmd), Unwrap(outw.bb), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
+                        Unwrap(outw.resolveimg), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &resolve);
+
+    // wait for resolve to finish before we blit
+    copySource = outw.resolveimg;
+
+    resolveBarrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
+    resolveBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
+    DoPipelineBarrier(cmd, 1, &resolveBarrier);
+  }
+
+  vt->CmdCopyImageToBuffer(Unwrap(cmd), Unwrap(copySource), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
                            readbackBuf, 1, &cpy);
 
   outw.bbBarrier.oldLayout = outw.bbBarrier.newLayout;
@@ -1088,7 +1126,6 @@ void VulkanReplay::FlipOutputWindow(uint64_t id)
 
   VkImage blitSource = outw.bb;
 
-#if ENABLED(MSAA_MESH_VIEW)
   if(outw.dsimg != VK_NULL_HANDLE)
   {
     VkImageResolve resolve = {
@@ -1123,7 +1160,6 @@ void VulkanReplay::FlipOutputWindow(uint64_t id)
     resolveBarrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
     DoPipelineBarrier(cmd, 1, &resolveBarrier);
   }
-#endif
 
   vt->CmdBlitImage(Unwrap(cmd), Unwrap(blitSource), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
                    Unwrap(outw.colimg[outw.curidx]), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit,

renderdoc/driver/vulkan/vk_replay.h

@@ -131,13 +131,7 @@
     msgprinted = true;                                   \
   } while((void)0, 0)
 
-#define MSAA_MESH_VIEW OPTION_ON
-
-#if ENABLED(MSAA_MESH_VIEW)
 #define VULKAN_MESH_VIEW_SAMPLES VK_SAMPLE_COUNT_4_BIT
-#else
-#define VULKAN_MESH_VIEW_SAMPLES VK_SAMPLE_COUNT_1_BIT
-#endif
 
 class AMDCounters;
 class WrappedVulkan;