Make minmax/histogram aware of custom shaders. Refs #385

* This moves those functions relative to an output instead of the
  renderer, so they pick up the settings etc from the output
  configuration.

renderdoc/api/replay/renderdoc_replay.h

@@ -162,6 +162,10 @@ struct IReplayOutput
   virtual bool SetPixelContextLocation(uint32_t x, uint32_t y) = 0;
   virtual void DisablePixelContext() = 0;
 
+  virtual bool GetMinMax(PixelValue *minval, PixelValue *maxval) = 0;
+  virtual bool GetHistogram(float minval, float maxval, bool channels[4],
+                            rdctype::array<uint32_t> *histogram) = 0;
+
   virtual bool PickPixel(ResourceId texID, bool customShader, uint32_t x, uint32_t y,
                          uint32_t sliceFace, uint32_t mip, uint32_t sample, PixelValue *val) = 0;
 };
@@ -205,6 +209,13 @@ extern "C" RENDERDOC_API void RENDERDOC_CC ReplayOutput_DisablePixelContext(Repl
 extern "C" RENDERDOC_API void RENDERDOC_CC ReplayOutput_GetCustomShaderTexID(ReplayOutput *output,
                                                                              ResourceId *id);
 
+extern "C" RENDERDOC_API bool32 RENDERDOC_CC ReplayOutput_GetMinMax(ReplayOutput *output,
+                                                                    PixelValue *minval,
+                                                                    PixelValue *maxval);
+extern "C" RENDERDOC_API bool32 RENDERDOC_CC
+ReplayOutput_GetHistogram(ReplayOutput *output, float minval, float maxval, bool32 channels[4],
+                          rdctype::array<uint32_t> *histogram);
+
 extern "C" RENDERDOC_API bool32 RENDERDOC_CC ReplayOutput_PickPixel(
     ReplayOutput *output, ResourceId texID, bool32 customShader, uint32_t x, uint32_t y,
     uint32_t sliceFace, uint32_t mip, uint32_t sample, PixelValue *val);
@@ -279,12 +290,6 @@ struct IReplayRenderer
 
   virtual bool GetPostVSData(uint32_t instID, MeshDataStage stage, MeshFormat *data) = 0;
 
-  virtual bool GetMinMax(ResourceId tex, uint32_t sliceFace, uint32_t mip, uint32_t sample,
-                         FormatComponentType typeHint, PixelValue *minval, PixelValue *maxval) = 0;
-  virtual bool GetHistogram(ResourceId tex, uint32_t sliceFace, uint32_t mip, uint32_t sample,
-                            FormatComponentType typeHint, float minval, float maxval,
-                            bool channels[4], rdctype::array<uint32_t> *histogram) = 0;
-
   virtual bool GetBufferData(ResourceId buff, uint64_t offset, uint64_t len,
                              rdctype::array<byte> *data) = 0;
   virtual bool GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
@@ -405,14 +410,6 @@ extern "C" RENDERDOC_API bool32 RENDERDOC_CC ReplayRenderer_GetPostVSData(Replay
                                                                           MeshDataStage stage,
                                                                           MeshFormat *data);
 
-extern "C" RENDERDOC_API bool32 RENDERDOC_CC ReplayRenderer_GetMinMax(
-    ReplayRenderer *rend, ResourceId tex, uint32_t sliceFace, uint32_t mip, uint32_t sample,
-    FormatComponentType typeHint, PixelValue *minval, PixelValue *maxval);
-extern "C" RENDERDOC_API bool32 RENDERDOC_CC
-ReplayRenderer_GetHistogram(ReplayRenderer *rend, ResourceId tex, uint32_t sliceFace, uint32_t mip,
-                            uint32_t sample, FormatComponentType typeHint, float minval,
-                            float maxval, bool32 channels[4], rdctype::array<uint32_t> *histogram);
-
 extern "C" RENDERDOC_API bool32 RENDERDOC_CC ReplayRenderer_GetBufferData(
     ReplayRenderer *rend, ResourceId buff, uint64_t offset, uint64_t len, rdctype::array<byte> *data);
 extern "C" RENDERDOC_API bool32 RENDERDOC_CC

renderdoc/replay/replay_output.cpp

@@ -292,6 +292,68 @@ bool ReplayOutput::AddThumbnail(WindowingSystem system, void *data, ResourceId t
   return true;
 }
 
+bool ReplayOutput::GetMinMax(PixelValue *minval, PixelValue *maxval)
+{
+  PixelValue *a = minval;
+  PixelValue *b = maxval;
+
+  PixelValue dummy;
+
+  if(a == NULL)
+    a = &dummy;
+  if(b == NULL)
+    b = &dummy;
+
+  ResourceId tex = m_pDevice->GetLiveID(m_RenderData.texDisplay.texid);
+
+  FormatComponentType typeHint = m_RenderData.texDisplay.typeHint;
+  uint32_t slice = m_RenderData.texDisplay.sliceFace;
+  uint32_t mip = m_RenderData.texDisplay.mip;
+  uint32_t sample = m_RenderData.texDisplay.sampleIdx;
+
+  if(m_RenderData.texDisplay.CustomShader != ResourceId() && m_CustomShaderResourceId != ResourceId())
+  {
+    tex = m_CustomShaderResourceId;
+    typeHint = eCompType_None;
+    slice = 0;
+    sample = 0;
+  }
+
+  return m_pDevice->GetMinMax(tex, slice, mip, sample, typeHint, &a->value_f[0], &b->value_f[0]);
+}
+
+bool ReplayOutput::GetHistogram(float minval, float maxval, bool channels[4],
+                                rdctype::array<uint32_t> *histogram)
+{
+  if(histogram == NULL)
+    return false;
+
+  vector<uint32_t> hist;
+
+  ResourceId tex = m_pDevice->GetLiveID(m_RenderData.texDisplay.texid);
+
+  FormatComponentType typeHint = m_RenderData.texDisplay.typeHint;
+  uint32_t slice = m_RenderData.texDisplay.sliceFace;
+  uint32_t mip = m_RenderData.texDisplay.mip;
+  uint32_t sample = m_RenderData.texDisplay.sampleIdx;
+
+  if(m_RenderData.texDisplay.CustomShader != ResourceId() && m_CustomShaderResourceId != ResourceId())
+  {
+    tex = m_CustomShaderResourceId;
+    typeHint = eCompType_None;
+    slice = 0;
+    sample = 0;
+  }
+
+  bool ret =
+      m_pDevice->GetHistogram(tex, slice, mip, sample, typeHint, minval, maxval, channels, hist);
+
+  if(ret)
+    *histogram = hist;
+
+  return ret;
+}
+
 bool ReplayOutput::PickPixel(ResourceId tex, bool customShader, uint32_t x, uint32_t y,
                              uint32_t sliceFace, uint32_t mip, uint32_t sample, PixelValue *ret)
 {
@@ -813,6 +875,20 @@ extern "C" RENDERDOC_API void RENDERDOC_CC ReplayOutput_GetCustomShaderTexID(Rep
     *id = output->GetCustomShaderTexID();
 }
 
+extern "C" RENDERDOC_API bool32 RENDERDOC_CC ReplayOutput_GetMinMax(ReplayOutput *output,
+                                                                    PixelValue *minval,
+                                                                    PixelValue *maxval)
+{
+  return output->GetMinMax(minval, maxval);
+}
+extern "C" RENDERDOC_API bool32 RENDERDOC_CC
+ReplayOutput_GetHistogram(ReplayOutput *output, float minval, float maxval, bool32 channels[4],
+                          rdctype::array<uint32_t> *histogram)
+{
+  bool chans[4] = {channels[0] != 0, channels[1] != 0, channels[2] != 0, channels[3] != 0};
+  return output->GetHistogram(minval, maxval, chans, histogram);
+}
+
 extern "C" RENDERDOC_API bool32 RENDERDOC_CC ReplayOutput_PickPixel(
     ReplayOutput *output, ResourceId texID, bool32 customShader, uint32_t x, uint32_t y,
     uint32_t sliceFace, uint32_t mip, uint32_t sample, PixelValue *val)

renderdoc/replay/replay_renderer.cpp

@@ -397,41 +397,6 @@ bool ReplayRenderer::GetPostVSData(uint32_t instID, MeshDataStage stage, MeshFor
   return true;
 }
 
-bool ReplayRenderer::GetMinMax(ResourceId tex, uint32_t sliceFace, uint32_t mip, uint32_t sample,
-                               FormatComponentType typeHint, PixelValue *minval, PixelValue *maxval)
-{
-  PixelValue *a = minval;
-  PixelValue *b = maxval;
-
-  PixelValue dummy;
-
-  if(a == NULL)
-    a = &dummy;
-  if(b == NULL)
-    b = &dummy;
-
-  return m_pDevice->GetMinMax(m_pDevice->GetLiveID(tex), sliceFace, mip, sample, typeHint,
-                              &a->value_f[0], &b->value_f[0]);
-}
-
-bool ReplayRenderer::GetHistogram(ResourceId tex, uint32_t sliceFace, uint32_t mip, uint32_t sample,
-                                  FormatComponentType typeHint, float minval, float maxval,
-                                  bool channels[4], rdctype::array<uint32_t> *histogram)
-{
-  if(histogram == NULL)
-    return false;
-
-  vector<uint32_t> hist;
-
-  bool ret = m_pDevice->GetHistogram(m_pDevice->GetLiveID(tex), sliceFace, mip, sample, typeHint,
-                                     minval, maxval, channels, hist);
-
-  if(ret)
-    *histogram = hist;
-
-  return ret;
-}
-
 bool ReplayRenderer::GetBufferData(ResourceId buff, uint64_t offset, uint64_t len,
                                    rdctype::array<byte> *data)
 {
@@ -1892,21 +1857,6 @@ extern "C" RENDERDOC_API bool32 RENDERDOC_CC ReplayRenderer_GetPostVSData(Replay
   return rend->GetPostVSData(instID, stage, data);
 }
 
-extern "C" RENDERDOC_API bool32 RENDERDOC_CC ReplayRenderer_GetMinMax(
-    ReplayRenderer *rend, ResourceId tex, uint32_t sliceFace, uint32_t mip, uint32_t sample,
-    FormatComponentType typeHint, PixelValue *minval, PixelValue *maxval)
-{
-  return rend->GetMinMax(tex, sliceFace, mip, sample, typeHint, minval, maxval);
-}
-extern "C" RENDERDOC_API bool32 RENDERDOC_CC
-ReplayRenderer_GetHistogram(ReplayRenderer *rend, ResourceId tex, uint32_t sliceFace, uint32_t mip,
-                            uint32_t sample, FormatComponentType typeHint, float minval,
-                            float maxval, bool32 channels[4], rdctype::array<uint32_t> *histogram)
-{
-  bool chans[4] = {channels[0] != 0, channels[1] != 0, channels[2] != 0, channels[3] != 0};
-  return rend->GetHistogram(tex, sliceFace, mip, sample, typeHint, minval, maxval, chans, histogram);
-}
-
 extern "C" RENDERDOC_API bool32 RENDERDOC_CC ReplayRenderer_GetBufferData(
     ReplayRenderer *rend, ResourceId buff, uint64_t offset, uint64_t len, rdctype::array<byte> *data)
 {

renderdoc/replay/replay_renderer.h

@@ -53,6 +53,10 @@ public:
   bool SetPixelContextLocation(uint32_t x, uint32_t y);
   void DisablePixelContext();
 
+  bool GetMinMax(PixelValue *minval, PixelValue *maxval);
+  bool GetHistogram(float minval, float maxval, bool channels[4],
+                    rdctype::array<uint32_t> *histogram);
+
   ResourceId GetCustomShaderTexID() { return m_CustomShaderResourceId; }
   bool PickPixel(ResourceId texID, bool customShader, uint32_t x, uint32_t y, uint32_t sliceFace,
                  uint32_t mip, uint32_t sample, PixelValue *val);
@@ -172,12 +176,6 @@ public:
 
   bool GetPostVSData(uint32_t instID, MeshDataStage stage, MeshFormat *data);
 
-  bool GetMinMax(ResourceId tex, uint32_t sliceFace, uint32_t mip, uint32_t sample,
-                 FormatComponentType typeHint, PixelValue *minval, PixelValue *maxval);
-  bool GetHistogram(ResourceId tex, uint32_t sliceFace, uint32_t mip, uint32_t sample,
-                    FormatComponentType typeHint, float minval, float maxval, bool channels[4],
-                    rdctype::array<uint32_t> *histogram);
-
   bool GetUsage(ResourceId id, rdctype::array<EventUsage> *usage);
 
   bool GetBufferData(ResourceId buff, uint64_t offset, uint64_t len, rdctype::array<byte> *data);

renderdocui/Interop/ReplayRenderer.cs

@@ -143,6 +143,11 @@ namespace renderdoc
         [DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
         private static extern void ReplayOutput_GetCustomShaderTexID(IntPtr real, ref ResourceId texid);
 
+        [DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
+        private static extern bool ReplayOutput_GetMinMax(IntPtr real, IntPtr outminval, IntPtr outmaxval);
+        [DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
+        private static extern bool ReplayOutput_GetHistogram(IntPtr real, float minval, float maxval, bool[] channels, IntPtr outhistogram);
+
         [DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
         private static extern bool ReplayOutput_PickPixel(IntPtr real, ResourceId texID, bool customShader,
                                                                 UInt32 x, UInt32 y, UInt32 sliceFace, UInt32 mip, UInt32 sample, IntPtr outval);
@@ -204,6 +209,50 @@ namespace renderdoc
             return ret;
         }
 
+        public bool GetMinMax(out PixelValue minval, out PixelValue maxval)
+        {
+            IntPtr mem1 = CustomMarshal.Alloc(typeof(PixelValue));
+            IntPtr mem2 = CustomMarshal.Alloc(typeof(PixelValue));
+
+            bool success = ReplayOutput_GetMinMax(m_Real, mem1, mem2);
+
+            if (success)
+            {
+                minval = (PixelValue)CustomMarshal.PtrToStructure(mem1, typeof(PixelValue), true);
+                maxval = (PixelValue)CustomMarshal.PtrToStructure(mem2, typeof(PixelValue), true);
+            }
+            else
+            {
+                minval = null;
+                maxval = null;
+            }
+
+            CustomMarshal.Free(mem1);
+            CustomMarshal.Free(mem2);
+
+            return success;
+        }
+
+        public bool GetHistogram(float minval, float maxval,
+                                 bool Red, bool Green, bool Blue, bool Alpha,
+                                 out UInt32[] histogram)
+        {
+            IntPtr mem = CustomMarshal.Alloc(typeof(templated_array));
+
+            bool[] channels = new bool[] { Red, Green, Blue, Alpha };
+
+            bool success = ReplayOutput_GetHistogram(m_Real, minval, maxval, channels, mem);
+
+            histogram = null;
+
+            if (success)
+                histogram = (UInt32[])CustomMarshal.GetTemplatedArray(mem, typeof(UInt32), true);
+
+            CustomMarshal.Free(mem);
+
+            return success;
+        }
+
         public PixelValue PickPixel(ResourceId texID, bool customShader, UInt32 x, UInt32 y, UInt32 sliceFace, UInt32 mip, UInt32 sample)
         {
             IntPtr mem = CustomMarshal.Alloc(typeof(PixelValue));
@@ -313,11 +362,6 @@ namespace renderdoc
         [DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
         private static extern bool ReplayRenderer_GetPostVSData(IntPtr real, UInt32 instID, MeshDataStage stage, IntPtr outdata);
 
-        [DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
-        private static extern bool ReplayRenderer_GetMinMax(IntPtr real, ResourceId tex, UInt32 sliceFace, UInt32 mip, UInt32 sample, FormatComponentType typeHint, IntPtr outminval, IntPtr outmaxval);
-        [DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
-        private static extern bool ReplayRenderer_GetHistogram(IntPtr real, ResourceId tex, UInt32 sliceFace, UInt32 mip, UInt32 sample, FormatComponentType typeHint, float minval, float maxval, bool[] channels, IntPtr outhistogram);
-
         [DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
         private static extern bool ReplayRenderer_GetBufferData(IntPtr real, ResourceId buff, UInt64 offset, UInt64 len, IntPtr outdata);
         [DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)]
@@ -821,50 +865,6 @@ namespace renderdoc
             return ret;
         }
 
-        public bool GetMinMax(ResourceId tex, UInt32 sliceFace, UInt32 mip, UInt32 sample, FormatComponentType typeHint, out PixelValue minval, out PixelValue maxval)
-        {
-            IntPtr mem1 = CustomMarshal.Alloc(typeof(PixelValue));
-            IntPtr mem2 = CustomMarshal.Alloc(typeof(PixelValue));
-
-            bool success = ReplayRenderer_GetMinMax(m_Real, tex, sliceFace, mip, sample, typeHint, mem1, mem2);
-
-            if (success)
-            {
-                minval = (PixelValue)CustomMarshal.PtrToStructure(mem1, typeof(PixelValue), true);
-                maxval = (PixelValue)CustomMarshal.PtrToStructure(mem2, typeof(PixelValue), true);
-            }
-            else
-            {
-                minval = null;
-                maxval = null;
-            }
-
-            CustomMarshal.Free(mem1);
-            CustomMarshal.Free(mem2);
-
-            return success;
-        }
-
-        public bool GetHistogram(ResourceId tex, UInt32 sliceFace, UInt32 mip, UInt32 sample, FormatComponentType typeHint, float minval, float maxval,
-                                 bool Red, bool Green, bool Blue, bool Alpha,
-                                 out UInt32[] histogram)
-        {
-            IntPtr mem = CustomMarshal.Alloc(typeof(templated_array));
-
-            bool[] channels = new bool[] { Red, Green, Blue, Alpha };
-
-            bool success = ReplayRenderer_GetHistogram(m_Real, tex, sliceFace, mip, sample, typeHint, minval, maxval, channels, mem);
-
-            histogram = null;
-
-            if (success)
-                histogram = (UInt32[])CustomMarshal.GetTemplatedArray(mem, typeof(UInt32), true);
-
-            CustomMarshal.Free(mem);
-
-            return success;
-        }
-
         public byte[] GetBufferData(ResourceId buff, UInt64 offset, UInt64 len)
         {
             IntPtr mem = CustomMarshal.Alloc(typeof(templated_array));

renderdocui/Windows/TextureViewer.cs

@@ -3331,16 +3331,13 @@ namespace renderdocui.Windows
         private void AutoFitRange()
         {
             // no log loaded or buffer/empty texture currently being viewed - don't autofit
-            if (!m_Core.LogLoaded || CurrentTexture == null)
+            if (!m_Core.LogLoaded || CurrentTexture == null || m_Output == null)
                 return;
 
             m_Core.Renderer.BeginInvoke((ReplayRenderer r) =>
             {
                 PixelValue min, max;
-                bool success = r.GetMinMax(m_TexDisplay.texid,
+                bool success = m_Output.GetMinMax(out min, out max);
-                    m_TexDisplay.sliceFace, m_TexDisplay.mip, m_TexDisplay.sampleIdx,
-                    m_TexDisplay.typeHint,
-                    out min, out max);
 
                 if (success)
                 {
@@ -3351,6 +3348,11 @@ namespace renderdocui.Windows
 
                     ResourceFormat fmt = CurrentTexture.format;
 
+                    if (m_TexDisplay.CustomShader != ResourceId.Null)
+                    {
+                        fmt.compType = FormatComponentType.Float;
+                    }
+
                     for (int i = 0; i < 4; i++)
                     {
                         if (fmt.compType == FormatComponentType.UInt)
@@ -3424,17 +3426,17 @@ namespace renderdocui.Windows
 
         private void RT_UpdateVisualRange(ReplayRenderer r)
         {
-            if (!m_Visualise || CurrentTexture == null) return;
+            if (!m_Visualise || CurrentTexture == null || m_Output == null) return;
 
             ResourceFormat fmt = CurrentTexture.format;
 
+            if (m_TexDisplay.CustomShader != ResourceId.Null)
+                fmt.compCount = 4;
+
             bool success = true;
 
             uint[] histogram;
-            success = r.GetHistogram(m_TexDisplay.texid,
+            success = m_Output.GetHistogram(rangeHistogram.RangeMin, rangeHistogram.RangeMax,
-                                     m_TexDisplay.sliceFace, m_TexDisplay.mip, m_TexDisplay.sampleIdx,
-                                     m_TexDisplay.typeHint,
-                                     rangeHistogram.RangeMin, rangeHistogram.RangeMax,
                                      m_TexDisplay.Red,
                                      m_TexDisplay.Green && fmt.compCount > 1,
                                      m_TexDisplay.Blue && fmt.compCount > 2,