From 4b830d4a663de4c9ac7d309117d5992aa143a88e Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Fri, 10 Apr 2020 17:57:10 +0100
Subject: [PATCH] Pass the texture type to CalculateSampleGather

---
 .../driver/shaders/spirv/spirv_debug.cpp      | 14 ++++++++++---
 renderdoc/driver/shaders/spirv/spirv_debug.h  | 21 +++++++++++++++----
 .../shaders/spirv/spirv_debug_setup.cpp       | 17 +++++++++++++++
 renderdoc/driver/vulkan/vk_shaderdebug.cpp    |  3 ++-
 4 files changed, 47 insertions(+), 8 deletions(-)

diff --git a/renderdoc/driver/shaders/spirv/spirv_debug.cpp b/renderdoc/driver/shaders/spirv/spirv_debug.cpp
index 1836963c9..441031127 100644
--- a/renderdoc/driver/shaders/spirv/spirv_debug.cpp
+++ b/renderdoc/driver/shaders/spirv/spirv_debug.cpp
@@ -1566,13 +1566,21 @@ void ThreadState::StepNext(ShaderDebugState *state,
       RDCASSERT(sampler.type == VarType::Unknown || sampler.type == VarType::ReadOnlyResource ||
                 sampler.type == VarType::Sampler);
 
+      // at setup time we stored the texture type for easy access here
+      DebugAPIWrapper::TextureType texType =
+          (DebugAPIWrapper::TextureType)img.value.uv[TextureTypeVariableSlot];
+
       ShaderVariable result;
 
       result.type = resultType.scalar().Type();
 
-      if(!debugger.GetAPIWrapper()->CalculateSampleGather(*this, opdata.op, img.GetBinding(),
-                                                          sampler.GetBinding(), uv, ddxCalc, ddyCalc,
-                                                          compare, gather, operands, result))
+      BindpointIndex samplerIndex = BindpointIndex(~0U, ~0U, ~0U);
+      if(sampler.type == VarType::Sampler)
+        samplerIndex = sampler.GetBinding();
+
+      if(!debugger.GetAPIWrapper()->CalculateSampleGather(
+             *this, opdata.op, texType, img.GetBinding(), samplerIndex, uv, ddxCalc, ddyCalc,
+             compare, gather, operands, result))
       {
         // sample failed. Pretend we got 0 columns back
 
diff --git a/renderdoc/driver/shaders/spirv/spirv_debug.h b/renderdoc/driver/shaders/spirv/spirv_debug.h
index 7c38c3786..92df5e10e 100644
--- a/renderdoc/driver/shaders/spirv/spirv_debug.h
+++ b/renderdoc/driver/shaders/spirv/spirv_debug.h
@@ -58,10 +58,21 @@ public:
   virtual void FillInputValue(ShaderVariable &var, ShaderBuiltin builtin, uint32_t location,
                               uint32_t component) = 0;
 
-  virtual bool CalculateSampleGather(ThreadState &thread, rdcspv::Op opcode, BindpointIndex imageBind,
-                                     BindpointIndex samplerBind, const ShaderVariable &uv,
-                                     const ShaderVariable &ddxCalc, const ShaderVariable &ddyCalc,
-                                     const ShaderVariable &compare, GatherChannel gatherChannel,
+  enum TextureType
+  {
+    Float_Texture = 0x00,
+
+    UInt_Texture = 0x01,
+    SInt_Texture = 0x02,
+
+    Buffer_Texture = 0x10,
+  };
+
+  virtual bool CalculateSampleGather(ThreadState &lane, rdcspv::Op opcode, TextureType texType,
+                                     BindpointIndex imageBind, BindpointIndex samplerBind,
+                                     const ShaderVariable &uv, const ShaderVariable &ddxCalc,
+                                     const ShaderVariable &ddyCalc, const ShaderVariable &compare,
+                                     GatherChannel gatherChannel,
                                      const ImageOperandsAndParamDatas &operands,
                                      ShaderVariable &output) = 0;
 
@@ -77,6 +88,8 @@ public:
                                          uint32_t component) = 0;
 };
 
+static const uint32_t TextureTypeVariableSlot = 8;
+
 typedef ShaderVariable (*ExtInstImpl)(ThreadState &, const rdcarray<Id> &);
 
 struct ExtInstDispatcher
diff --git a/renderdoc/driver/shaders/spirv/spirv_debug_setup.cpp b/renderdoc/driver/shaders/spirv/spirv_debug_setup.cpp
index 2917be46c..8d49fc534 100644
--- a/renderdoc/driver/shaders/spirv/spirv_debug_setup.cpp
+++ b/renderdoc/driver/shaders/spirv/spirv_debug_setup.cpp
@@ -406,6 +406,23 @@ ShaderDebugTrace *Debugger::BeginDebug(DebugAPIWrapper *apiWrapper, const Shader
         var.type = VarType::ReadOnlyResource;
         debugType = DebugVariableType::ReadOnlyResource;
 
+        // store the texture type here, since the image may be copied around and combined with a
+        // sampler, so accessing the original type might be non-trivial at point of access
+        uint32_t texType = DebugAPIWrapper::Float_Texture;
+
+        if(imageTypes[type.InnerType()].dim == Dim::Buffer)
+          texType |= DebugAPIWrapper::Buffer_Texture;
+
+        if(imageTypes[type.InnerType()].retType.type == Op::TypeInt)
+        {
+          if(imageTypes[type.InnerType()].retType.signedness)
+            texType |= DebugAPIWrapper::SInt_Texture;
+          else
+            texType |= DebugAPIWrapper::UInt_Texture;
+        }
+
+        var.value.uv[TextureTypeVariableSlot] = texType;
+
         global.readOnlyResources.push_back(var);
         readOnlyIDs.push_back(v.id);
       }
diff --git a/renderdoc/driver/vulkan/vk_shaderdebug.cpp b/renderdoc/driver/vulkan/vk_shaderdebug.cpp
index 2d71f2156..177626bb0 100644
--- a/renderdoc/driver/vulkan/vk_shaderdebug.cpp
+++ b/renderdoc/driver/vulkan/vk_shaderdebug.cpp
@@ -266,7 +266,8 @@ public:
     return DerivativeDeltas();
   }
 
-  bool CalculateSampleGather(rdcspv::ThreadState &thread, rdcspv::Op opcode, BindpointIndex imageBind,
+  bool CalculateSampleGather(rdcspv::ThreadState &lane, rdcspv::Op opcode,
+                             DebugAPIWrapper::TextureType texType, BindpointIndex imageBind,
                              BindpointIndex samplerBind, const ShaderVariable &uv,
                              const ShaderVariable &ddxCalc, const ShaderVariable &ddyCalc,
                              const ShaderVariable &compare, rdcspv::GatherChannel gatherChannel,