From 22203fbb0270490e9394f2d192272042d80f1d28 Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Wed, 26 Jun 2019 12:42:18 +0100
Subject: [PATCH] Add checks for quad overdraw and triangle size overlays on
 mac

* These features aren't supported by the underlying implementation.
* Triangle size we can feature-detect properly - by checking for geometry shader
  support. Quad resolve should be supported but metal doesn't allow atomics on
  images so we have to check specifically for a metal backend.
---
 renderdoc/driver/vulkan/vk_debug.cpp        |  6 +++++-
 renderdoc/driver/vulkan/vk_overlay.cpp      | 20 +++++++++++---------
 renderdoc/driver/vulkan/vk_shader_cache.cpp |  2 +-
 3 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/renderdoc/driver/vulkan/vk_debug.cpp b/renderdoc/driver/vulkan/vk_debug.cpp
index c47ea62ac..7862874e7 100644
--- a/renderdoc/driver/vulkan/vk_debug.cpp
+++ b/renderdoc/driver/vulkan/vk_debug.cpp
@@ -2122,7 +2122,11 @@ void VulkanReplay::OverlayRendering::Init(WrappedVulkan *driver, VkDescriptorPoo
     pipeInfo.fragment = shaderCache->GetBuiltinModule(BuiltinShader::QuadResolveFS);
     pipeInfo.pipeLayout = m_QuadResolvePipeLayout;
 
-    CREATE_OBJECT(m_QuadResolvePipeline[i], pipeInfo);
+    if(pipeInfo.fragment != VK_NULL_HANDLE &&
+       shaderCache->GetBuiltinModule(BuiltinShader::QuadWriteFS) != VK_NULL_HANDLE)
+    {
+      CREATE_OBJECT(m_QuadResolvePipeline[i], pipeInfo);
+    }
 
     driver->vkDestroyRenderPass(driver->GetDev(), RGBA16MSRP, NULL);
   }
diff --git a/renderdoc/driver/vulkan/vk_overlay.cpp b/renderdoc/driver/vulkan/vk_overlay.cpp
index dccb4bfa4..93f01b2fd 100644
--- a/renderdoc/driver/vulkan/vk_overlay.cpp
+++ b/renderdoc/driver/vulkan/vk_overlay.cpp
@@ -1870,6 +1870,17 @@ ResourceId VulkanReplay::RenderOverlay(ResourceId texid, CompType typeHint, Debu
   {
     VulkanRenderState prevstate = m_pDriver->m_RenderState;
 
+    VkPipelineShaderStageCreateInfo stages[3] = {
+        {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, NULL, 0, VK_SHADER_STAGE_VERTEX_BIT,
+         shaderCache->GetBuiltinModule(BuiltinShader::MeshVS), "main", NULL},
+        {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, NULL, 0, VK_SHADER_STAGE_FRAGMENT_BIT,
+         shaderCache->GetBuiltinModule(BuiltinShader::TrisizeFS), "main", NULL},
+        {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, NULL, 0, VK_SHADER_STAGE_GEOMETRY_BIT,
+         shaderCache->GetBuiltinModule(BuiltinShader::TrisizeGS), "main", NULL},
+    };
+
+    if(stages[0].module != VK_NULL_HANDLE && stages[1].module != VK_NULL_HANDLE &&
+       stages[2].module != VK_NULL_HANDLE)
     {
       SCOPED_TIMER("Triangle Size");
 
@@ -2076,15 +2087,6 @@ ResourceId VulkanReplay::RenderOverlay(ResourceId texid, CompType typeHint, Debu
 
       m_pDriver->GetShaderCache()->MakeGraphicsPipelineInfo(pipeCreateInfo, state.graphics.pipeline);
 
-      VkPipelineShaderStageCreateInfo stages[3] = {
-          {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, NULL, 0, VK_SHADER_STAGE_VERTEX_BIT,
-           shaderCache->GetBuiltinModule(BuiltinShader::MeshVS), "main", NULL},
-          {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, NULL, 0, VK_SHADER_STAGE_FRAGMENT_BIT,
-           shaderCache->GetBuiltinModule(BuiltinShader::TrisizeFS), "main", NULL},
-          {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, NULL, 0, VK_SHADER_STAGE_GEOMETRY_BIT,
-           shaderCache->GetBuiltinModule(BuiltinShader::TrisizeGS), "main", NULL},
-      };
-
       VkPipelineInputAssemblyStateCreateInfo ia = {
           VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO};
       ia.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
diff --git a/renderdoc/driver/vulkan/vk_shader_cache.cpp b/renderdoc/driver/vulkan/vk_shader_cache.cpp
index 91596e4aa..389b6ba4f 100644
--- a/renderdoc/driver/vulkan/vk_shader_cache.cpp
+++ b/renderdoc/driver/vulkan/vk_shader_cache.cpp
@@ -70,7 +70,7 @@ static const BuiltinShaderConfig builtinShaders[] = {
     {BuiltinShader::QuadResolveFS, EmbeddedResource(glsl_quadresolve_frag),
      SPIRVShaderStage::Fragment, FeatureCheck::FragmentStores, true},
     {BuiltinShader::QuadWriteFS, EmbeddedResource(glsl_quadwrite_frag), SPIRVShaderStage::Fragment,
-     FeatureCheck::FragmentStores, false},
+     FeatureCheck::FragmentStores | FeatureCheck::NonMetalBackend, false},
     {BuiltinShader::TrisizeGS, EmbeddedResource(glsl_trisize_geom), SPIRVShaderStage::Geometry,
      FeatureCheck::NoCheck, true},
     {BuiltinShader::TrisizeFS, EmbeddedResource(glsl_trisize_frag), SPIRVShaderStage::Fragment,