Fix missing stencil from image on vulkan when saving to disk

renderdoc/driver/vulkan/vk_debug.cpp

@@ -255,6 +255,7 @@ struct ConciseGraphicsPipeline
   bool blendEnable;
   VkBlendFactor srcBlend;
   VkBlendFactor dstBlend;
+  uint32_t writeMask;
 };
 
 static void create(WrappedVulkan *driver, const char *objName, const int line, VkPipeline *pipe,
@@ -315,7 +316,7 @@ static void create(WrappedVulkan *driver, const char *objName, const int line, V
       // alpha blending
       info.srcBlend, info.dstBlend, VK_BLEND_OP_ADD,
       // write mask
-      0xf,
+      info.writeMask,
   };
 
   const VkPipelineColorBlendStateCreateInfo colorBlend = {
@@ -512,6 +513,7 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver)
         false,    // blendEnable
         VK_BLEND_FACTOR_ONE,
         VK_BLEND_FACTOR_ZERO,
+        0xf,    // writeMask
     };
 
     CREATE_OBJECT(m_DepthMS2ArrayPipe[f], depthPipeInfo);
@@ -776,6 +778,7 @@ void VulkanDebugManager::CreateCustomShaderPipeline(ResourceId shader, VkPipelin
       false,    // blendEnable
       VK_BLEND_FACTOR_ONE,
       VK_BLEND_FACTOR_ZERO,
+      0xf,    // writeMask
   };
 
   CREATE_OBJECT(m_Custom.TexPipeline, customPipe);
@@ -1568,6 +1571,7 @@ void VulkanReplay::TextureRendering::Init(WrappedVulkan *driver, VkDescriptorPoo
         false,    // blendEnable
         VK_BLEND_FACTOR_ONE,
         VK_BLEND_FACTOR_ZERO,
+        0xf,    // writeMask
     };
 
     CREATE_OBJECT(Pipeline, texDisplayInfo);
@@ -1582,9 +1586,20 @@ void VulkanReplay::TextureRendering::Init(WrappedVulkan *driver, VkDescriptorPoo
     texDisplayInfo.blendEnable = true;
     texDisplayInfo.srcBlend = VK_BLEND_FACTOR_SRC_ALPHA;
     texDisplayInfo.dstBlend = VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA;
-
     CREATE_OBJECT(BlendPipeline, texDisplayInfo);
 
+    // make versions that only write to green, for doing two-pass stencil writes
+    texDisplayInfo.writeMask = 0x2;
+
+    texDisplayInfo.renderPass = SRGBA8RP;
+    CREATE_OBJECT(PipelineGreenOnly, texDisplayInfo);
+
+    texDisplayInfo.renderPass = RGBA32RP;
+    CREATE_OBJECT(F32PipelineGreenOnly, texDisplayInfo);
+
+    texDisplayInfo.renderPass = RGBA16RP;
+    CREATE_OBJECT(F16PipelineGreenOnly, texDisplayInfo);
+
     driver->vkDestroyRenderPass(driver->GetDev(), SRGBA8RP, NULL);
     driver->vkDestroyRenderPass(driver->GetDev(), RGBA16RP, NULL);
     driver->vkDestroyRenderPass(driver->GetDev(), RGBA32RP, NULL);
@@ -1774,6 +1789,10 @@ void VulkanReplay::TextureRendering::Destroy(WrappedVulkan *driver)
   driver->vkDestroyPipeline(driver->GetDev(), BlendPipeline, NULL);
   driver->vkDestroyPipeline(driver->GetDev(), F16Pipeline, NULL);
   driver->vkDestroyPipeline(driver->GetDev(), F32Pipeline, NULL);
+
+  driver->vkDestroyPipeline(driver->GetDev(), PipelineGreenOnly, NULL);
+  driver->vkDestroyPipeline(driver->GetDev(), F16PipelineGreenOnly, NULL);
+  driver->vkDestroyPipeline(driver->GetDev(), F32PipelineGreenOnly, NULL);
   UBO.Destroy();
 
   driver->vkDestroySampler(driver->GetDev(), LinearSampler, NULL);
@@ -1834,6 +1853,7 @@ void VulkanReplay::OverlayRendering::Init(WrappedVulkan *driver, VkDescriptorPoo
       true,    // blendEnable
       VK_BLEND_FACTOR_SRC_ALPHA,
       VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA,
+      0xf,    // writeMask
   };
 
   uint32_t samplesHandled = 0;
@@ -1976,6 +1996,7 @@ void VulkanReplay::CheckerboardRendering::Init(WrappedVulkan *driver, VkDescript
       false,    // blendEnable
       VK_BLEND_FACTOR_ONE,
       VK_BLEND_FACTOR_ZERO,
+      0xf,    // writeMask
   };
 
   CREATE_OBJECT(Pipeline, checkerInfo);

renderdoc/driver/vulkan/vk_rendertexture.cpp

@@ -119,6 +119,7 @@ bool VulkanReplay::RenderTextureInternal(TextureDisplay cfg, VkRenderPassBeginIn
   const bool blendAlpha = (flags & eTexDisplay_BlendAlpha) != 0;
   const bool mipShift = (flags & eTexDisplay_MipShift) != 0;
   const bool f16render = (flags & eTexDisplay_F16Render) != 0;
+  const bool greenonly = (flags & eTexDisplay_GreenOnly) != 0;
   const bool f32render = (flags & eTexDisplay_F32Render) != 0;
 
   VkDevice dev = m_pDriver->GetDev();
@@ -412,7 +413,7 @@ bool VulkanReplay::RenderTextureInternal(TextureDisplay cfg, VkRenderPassBeginIn
   {
     vt->CmdBeginRenderPass(Unwrap(cmd), &rpbegin, VK_SUBPASS_CONTENTS_INLINE);
 
-    VkPipeline pipe = m_TexRender.Pipeline;
+    VkPipeline pipe = greenonly ? m_TexRender.PipelineGreenOnly : m_TexRender.Pipeline;
 
     if(cfg.customShaderId != ResourceId())
     {
@@ -421,11 +422,11 @@ bool VulkanReplay::RenderTextureInternal(TextureDisplay cfg, VkRenderPassBeginIn
     }
     else if(f16render)
     {
-      pipe = m_TexRender.F16Pipeline;
+      pipe = greenonly ? m_TexRender.F16PipelineGreenOnly : m_TexRender.F16Pipeline;
     }
     else if(f32render)
     {
-      pipe = m_TexRender.F32Pipeline;
+      pipe = greenonly ? m_TexRender.F32PipelineGreenOnly : m_TexRender.F32Pipeline;
     }
     else if(!cfg.rawOutput && blendAlpha && cfg.customShaderId == ResourceId())
     {

renderdoc/driver/vulkan/vk_replay.cpp

@@ -2347,6 +2347,13 @@ void VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
       };
 
       RenderTextureInternal(texDisplay, rpbegin, renderFlags);
+
+      // for textures with stencil, do another draw to copy the stencil
+      if(isStencil)
+      {
+        texDisplay.red = texDisplay.blue = texDisplay.alpha = false;
+        RenderTextureInternal(texDisplay, rpbegin, renderFlags | eTexDisplay_GreenOnly);
+      }
     }
 
     m_DebugWidth = oldW;

renderdoc/driver/vulkan/vk_replay.h

@@ -423,6 +423,7 @@ private:
     eTexDisplay_F32Render = 0x2,
     eTexDisplay_BlendAlpha = 0x4,
     eTexDisplay_MipShift = 0x8,
+    eTexDisplay_GreenOnly = 0x10,
   };
 
   // General use/misc items that are used in many places
@@ -446,6 +447,10 @@ private:
     VkPipeline BlendPipeline = VK_NULL_HANDLE;
     VkPipeline F16Pipeline = VK_NULL_HANDLE;
     VkPipeline F32Pipeline = VK_NULL_HANDLE;
+
+    VkPipeline PipelineGreenOnly = VK_NULL_HANDLE;
+    VkPipeline F16PipelineGreenOnly = VK_NULL_HANDLE;
+    VkPipeline F32PipelineGreenOnly = VK_NULL_HANDLE;
     GPUBuffer UBO;
 
     // ring buffered to allow multiple texture renders between flushes