From 2d51d890e483b70b31a43941592f1c8ce88133bb Mon Sep 17 00:00:00 2001
From: Jake Turner <jake@evansturner.co.uk>
Date: Wed, 6 Dec 2023 17:53:38 +0000
Subject: [PATCH] Vk Depth overlay pipeline creation improvements

Use framebufferDepthSampleCounts instead of framebufferColorSampleCounts
Query for format support before trying to create a RenderPass using the format
---
 renderdoc/driver/vulkan/vk_debug.cpp | 40 ++++++++++++++++++++--------
 1 file changed, 29 insertions(+), 11 deletions(-)

diff --git a/renderdoc/driver/vulkan/vk_debug.cpp b/renderdoc/driver/vulkan/vk_debug.cpp
index a13b9b44a..20173b1b9 100644
--- a/renderdoc/driver/vulkan/vk_debug.cpp
+++ b/renderdoc/driver/vulkan/vk_debug.cpp
@@ -3968,13 +3968,13 @@ void VulkanReplay::OverlayRendering::Init(WrappedVulkan *driver, VkDescriptorPoo
   RDCCOMPILE_ASSERT(ARRAY_COUNT(m_CheckerF16Pipeline) == ARRAY_COUNT(m_QuadResolvePipeline),
                     "Arrays are mismatched in size!");
 
-  uint32_t supportedSampleCounts = driver->GetDeviceProps().limits.framebufferColorSampleCounts;
+  uint32_t supportedColorSampleCounts = driver->GetDeviceProps().limits.framebufferColorSampleCounts;
 
   for(size_t i = 0; i < ARRAY_COUNT(m_CheckerF16Pipeline); i++)
   {
     VkSampleCountFlagBits samples = VkSampleCountFlagBits(1 << i);
 
-    if((supportedSampleCounts & (uint32_t)samples) == 0)
+    if((supportedColorSampleCounts & (uint32_t)samples) == 0)
       continue;
 
     VkRenderPass RGBA16MSRP = VK_NULL_HANDLE;
@@ -4010,6 +4010,10 @@ void VulkanReplay::OverlayRendering::Init(WrappedVulkan *driver, VkDescriptorPoo
 
     driver->vkDestroyRenderPass(driver->GetDev(), RGBA16MSRP, NULL);
   }
+  RDCASSERTEQUAL((uint32_t)driver->GetDeviceProps().limits.framebufferColorSampleCounts,
+                 samplesHandled);
+
+  uint32_t supportedDepthSampleCounts = driver->GetDeviceProps().limits.framebufferDepthSampleCounts;
 
   samplesHandled = 0;
   {
@@ -4080,21 +4084,28 @@ void VulkanReplay::OverlayRendering::Init(WrappedVulkan *driver, VkDescriptorPoo
         NULL,    // dependencies
     };
 
-    RDCCOMPILE_ASSERT(ARRAY_COUNT(m_DepthResolvePipeline) == ARRAY_COUNT(m_DepthResolvePipeline),
+    RDCCOMPILE_ASSERT(ARRAY_COUNT(m_DepthCopyPipeline) == ARRAY_COUNT(m_DepthResolvePipeline),
                       "m_DepthCopyPipeline size must match m_DepthResolvePipeline");
 
     if(DepthCopyPipeInfo.fragment != VK_NULL_HANDLE)
     {
       for(size_t f = 0; f < ARRAY_COUNT(m_DepthCopyPipeline); ++f)
       {
+        for(size_t i = 0; i < ARRAY_COUNT(m_DepthCopyPipeline[f]); ++i)
+          m_DepthCopyPipeline[f][i] = VK_NULL_HANDLE;
+
         VkFormat fmt = (f == 0) ? VK_FORMAT_D24_UNORM_S8_UINT : VK_FORMAT_D32_SFLOAT_S8_UINT;
+        VkImageFormatProperties props;
+        if(driver->vkGetPhysicalDeviceImageFormatProperties(
+               driver->GetPhysDev(), fmt, VK_IMAGE_TYPE_2D, VK_IMAGE_TILING_OPTIMAL,
+               VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT, 0, &props) != VK_SUCCESS)
+          continue;
+
         attDescs[1].format = fmt;
         for(size_t i = 0; i < ARRAY_COUNT(m_DepthCopyPipeline[f]); ++i)
         {
-          m_DepthCopyPipeline[f][i] = VK_NULL_HANDLE;
           VkSampleCountFlagBits samples = VkSampleCountFlagBits(1 << i);
-
-          if((supportedSampleCounts & (uint32_t)samples) == 0)
+          if((supportedDepthSampleCounts & (uint32_t)samples) == 0)
             continue;
 
           VkRenderPass depthMSRP = VK_NULL_HANDLE;
@@ -4125,7 +4136,7 @@ void VulkanReplay::OverlayRendering::Init(WrappedVulkan *driver, VkDescriptorPoo
       }
     }
   }
-  RDCASSERTEQUAL((uint32_t)driver->GetDeviceProps().limits.framebufferColorSampleCounts,
+  RDCASSERTEQUAL((uint32_t)driver->GetDeviceProps().limits.framebufferDepthSampleCounts,
                  samplesHandled);
 
   samplesHandled = 0;
@@ -4208,14 +4219,21 @@ void VulkanReplay::OverlayRendering::Init(WrappedVulkan *driver, VkDescriptorPoo
     {
       for(size_t f = 0; f < ARRAY_COUNT(m_DepthResolvePipeline); ++f)
       {
+        for(size_t i = 0; i < ARRAY_COUNT(m_DepthResolvePipeline[f]); ++i)
+          m_DepthResolvePipeline[f][i] = VK_NULL_HANDLE;
+
         VkFormat fmt = (f == 0) ? VK_FORMAT_D24_UNORM_S8_UINT : VK_FORMAT_D32_SFLOAT_S8_UINT;
+        VkImageFormatProperties props;
+        if(driver->vkGetPhysicalDeviceImageFormatProperties(
+               driver->GetPhysDev(), fmt, VK_IMAGE_TYPE_2D, VK_IMAGE_TILING_OPTIMAL,
+               VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT, 0, &props) != VK_SUCCESS)
+          continue;
+
         attDescs[1].format = fmt;
         for(size_t i = 0; i < ARRAY_COUNT(m_DepthResolvePipeline[f]); ++i)
         {
-          m_DepthResolvePipeline[f][i] = VK_NULL_HANDLE;
           VkSampleCountFlagBits samples = VkSampleCountFlagBits(1 << i);
-
-          if((supportedSampleCounts & (uint32_t)samples) == 0)
+          if((supportedDepthSampleCounts & (uint32_t)samples) == 0)
             continue;
 
           VkRenderPass rgba16MSRP = VK_NULL_HANDLE;
@@ -4242,7 +4260,7 @@ void VulkanReplay::OverlayRendering::Init(WrappedVulkan *driver, VkDescriptorPoo
     }
   }
 
-  RDCASSERTEQUAL((uint32_t)driver->GetDeviceProps().limits.framebufferColorSampleCounts,
+  RDCASSERTEQUAL((uint32_t)driver->GetDeviceProps().limits.framebufferDepthSampleCounts,
                  samplesHandled);
 
   m_DefaultDepthStencilFormat = VK_FORMAT_UNDEFINED;