From 8a9a07d8a5d3c21890cbc285c31fb99749134fef Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Wed, 5 Sep 2018 14:52:53 +0100
Subject: [PATCH] Read stencil in red component on vulkan, don't use swizzles

---
 renderdoc/driver/vulkan/vk_rendertexture.cpp | 19 ++++---------------
 renderdoc/driver/vulkan/vk_replay.cpp        |  2 +-
 2 files changed, 5 insertions(+), 16 deletions(-)

diff --git a/renderdoc/driver/vulkan/vk_rendertexture.cpp b/renderdoc/driver/vulkan/vk_rendertexture.cpp
index 4e646911a..45bb1ffd3 100644
--- a/renderdoc/driver/vulkan/vk_rendertexture.cpp
+++ b/renderdoc/driver/vulkan/vk_rendertexture.cpp
@@ -67,21 +67,6 @@ void VulkanReplay::CreateTexImageView(VkImageAspectFlags aspectFlags, VkImage li
   if(iminfo.type == VK_IMAGE_TYPE_3D)
     viewInfo.viewType = VK_IMAGE_VIEW_TYPE_3D;
 
-  if(aspectFlags == VK_IMAGE_ASPECT_DEPTH_BIT)
-  {
-    viewInfo.components.r = VK_COMPONENT_SWIZZLE_R;
-    viewInfo.components.g = VK_COMPONENT_SWIZZLE_ZERO;
-    viewInfo.components.b = VK_COMPONENT_SWIZZLE_ZERO;
-    viewInfo.components.a = VK_COMPONENT_SWIZZLE_ZERO;
-  }
-  else if(aspectFlags == VK_IMAGE_ASPECT_STENCIL_BIT)
-  {
-    viewInfo.components.r = VK_COMPONENT_SWIZZLE_ZERO;
-    viewInfo.components.g = VK_COMPONENT_SWIZZLE_R;
-    viewInfo.components.b = VK_COMPONENT_SWIZZLE_ZERO;
-    viewInfo.components.a = VK_COMPONENT_SWIZZLE_ZERO;
-  }
-
   VkImageView view;
 
   VkResult vkr = ObjDisp(dev)->CreateImageView(Unwrap(dev), &viewInfo, NULL, &view);
@@ -180,6 +165,10 @@ bool VulkanReplay::RenderTextureInternal(TextureDisplay cfg, VkRenderPassBeginIn
       // rescale the range so that stencil seems to fit to 0-1
       cfg.rangeMin *= 255.0f;
       cfg.rangeMax *= 255.0f;
+
+      // shuffle the channel selection, since stencil comes back in red
+      cfg.red = true;
+      cfg.green = false;
     }
   }
 
diff --git a/renderdoc/driver/vulkan/vk_replay.cpp b/renderdoc/driver/vulkan/vk_replay.cpp
index 04c8398f2..06ca4c821 100644
--- a/renderdoc/driver/vulkan/vk_replay.cpp
+++ b/renderdoc/driver/vulkan/vk_replay.cpp
@@ -549,7 +549,7 @@ void VulkanReplay::PickPixel(ResourceId texture, uint32_t x, uint32_t y, uint32_
       // only write stencil to .y
       if(pass == 1)
       {
-        pixel[1] = ((uint32_t *)pData)[1] / 255.0f;
+        pixel[1] = ((uint32_t *)pData)[0] / 255.0f;
       }
       else
       {