Fix min/max calculation for stencil on GL and Vulkan

* This needs a separate pass for the stencil because GL/Vulkan treat stencil separately as a uint, whereas on D3D we can get away with binding a combined view and the stencil comes out as normalised floats.
2026-08-18 04:26:49 +00:00 · 2018-12-14 16:57:55 +00:00
parent aae08060a9
commit 8b04c9b7ad
4 changed files with 253 additions and 28 deletions
@@ -892,6 +892,60 @@ void GLReplay::DeleteDebugData()

 bool GLReplay::GetMinMax(ResourceId texid, uint32_t sliceFace, uint32_t mip, uint32_t sample,
                         CompType typeHint, float *minval, float *maxval)
+{
+  auto &texDetails = m_pDriver->m_Textures[texid];
+
+  if(GetBaseFormat(texDetails.internalFormat) == eGL_DEPTH_STENCIL)
+  {
+    // for depth/stencil we need to run the code twice - once to fetch depth and once to fetch
+    // stencil - since we can't process float depth and int stencil at the same time
+    Vec4f depth[2] = {
+        {0.0f, 0.0f, 0.0f, 0.0f}, {1.0f, 1.0f, 1.0f, 1.0f},
+    };
+    Vec4u stencil[2] = {{0, 0, 0, 0}, {1, 1, 1, 1}};
+
+    bool success =
+        GetMinMax(texid, sliceFace, mip, sample, typeHint, false, &depth[0].x, &depth[1].x);
+
+    if(!success)
+      return false;
+
+    success = GetMinMax(texid, sliceFace, mip, sample, typeHint, true, (float *)&stencil[0].x,
+                        (float *)&stencil[1].x);
+
+    if(!success)
+      return false;
+
+    // copy across into green channel, casting up to float, dividing by the range for this texture
+    float rangeScale = 1.0f;
+    switch(texDetails.internalFormat)
+    {
+      case eGL_STENCIL_INDEX1: rangeScale = 1.0f; break;
+      case eGL_STENCIL_INDEX4: rangeScale = 16.0f; break;
+      default:
+        RDCWARN("Unexpected raw format for stencil visualization");
+      // fall through
+      case eGL_DEPTH24_STENCIL8:
+      case eGL_DEPTH32F_STENCIL8:
+      case eGL_DEPTH_STENCIL:
+      case eGL_STENCIL_INDEX8: rangeScale = 255.0f; break;
+      case eGL_STENCIL_INDEX16: rangeScale = 65535.0f; break;
+    }
+
+    depth[0].y = float(stencil[0].x) / rangeScale;
+    depth[1].y = float(stencil[1].x) / rangeScale;
+
+    memcpy(minval, &depth[0].x, sizeof(depth[0]));
+    memcpy(maxval, &depth[1].x, sizeof(depth[1]));
+
+    return true;
+  }
+
+  return GetMinMax(texid, sliceFace, mip, sample, typeHint, false, minval, maxval);
+}
+
+bool GLReplay::GetMinMax(ResourceId texid, uint32_t sliceFace, uint32_t mip, uint32_t sample,
+                         CompType typeHint, bool stencil, float *minval, float *maxval)
 {
  if(texid == ResourceId() || m_pDriver->m_Textures.find(texid) == m_pDriver->m_Textures.end())
    return false;
@@ -959,6 +1013,27 @@ bool GLReplay::GetMinMax(ResourceId texid, uint32_t sliceFace, uint32_t mip, uin

  MakeCurrentReplayContext(m_DebugCtx);

+  RDCGLenum dsTexMode = eGL_NONE;
+  if(IsDepthStencilFormat(texDetails.internalFormat))
+  {
+    if(stencil)
+    {
+      dsTexMode = eGL_STENCIL_INDEX;
+      intIdx = 1;
+    }
+    else
+    {
+      dsTexMode = eGL_DEPTH_COMPONENT;
+    }
+  }
+  else
+  {
+    if(details.format.compType == CompType::UInt)
+      intIdx = 1;
+    if(details.format.compType == CompType::SInt)
+      intIdx = 2;
+  }
+
  GL.glBindBufferBase(eGL_UNIFORM_BUFFER, 2, DebugData.UBOs[0]);
  HistogramUBOData *cdata =
      (HistogramUBOData *)GL.glMapBufferRange(eGL_UNIFORM_BUFFER, 0, sizeof(HistogramUBOData),
@@ -982,16 +1057,10 @@ bool GLReplay::GetMinMax(ResourceId texid, uint32_t sliceFace, uint32_t mip, uin

  int progIdx = texSlot;

-  if(details.format.compType == CompType::UInt)
-  {
+  if(intIdx == 1)
    progIdx |= TEXDISPLAY_UINT_TEX;
-    intIdx = 1;
-  }
-  if(details.format.compType == CompType::SInt)
-  {
+  if(intIdx == 2)
    progIdx |= TEXDISPLAY_SINT_TEX;
-    intIdx = 2;
-  }

  int blocksX = (int)ceil(cdata->HistogramTextureResolution.x /
                          float(HGRAM_PIXELS_PER_TILE * HGRAM_TILES_PER_BLOCK));
@@ -1007,6 +1076,13 @@ bool GLReplay::GetMinMax(ResourceId texid, uint32_t sliceFace, uint32_t mip, uin
  else
    GL.glBindSampler(texSlot, DebugData.pointSampler);

+  GLint origDSTexMode = eGL_DEPTH_COMPONENT;
+  if(dsTexMode != eGL_NONE && HasExt[ARB_stencil_texturing])
+  {
+    GL.glGetTexParameteriv(target, eGL_DEPTH_STENCIL_TEXTURE_MODE, &origDSTexMode);
+    GL.glTexParameteri(target, eGL_DEPTH_STENCIL_TEXTURE_MODE, dsTexMode);
+  }
+
  int maxlevel = -1;

  int clampmaxlevel = details.mips - 1;
@@ -1055,6 +1131,9 @@ bool GLReplay::GetMinMax(ResourceId texid, uint32_t sliceFace, uint32_t mip, uin
  maxval[2] = minmax[1].z;
  maxval[3] = minmax[1].w;

+  if(dsTexMode != eGL_NONE && HasExt[ARB_stencil_texturing])
+    GL.glTexParameteri(target, eGL_DEPTH_STENCIL_TEXTURE_MODE, origDSTexMode);
+
  return true;
 }

@@ -1131,6 +1210,63 @@ bool GLReplay::GetHistogram(ResourceId texid, uint32_t sliceFace, uint32_t mip,

  MakeCurrentReplayContext(m_DebugCtx);

+  RDCGLenum dsTexMode = eGL_NONE;
+  if(IsDepthStencilFormat(texDetails.internalFormat))
+  {
+    // stencil-only, make sure we display it as such
+    if(texDetails.internalFormat == eGL_STENCIL_INDEX8)
+    {
+      channels[0] = false;
+      channels[1] = true;
+      channels[2] = false;
+      channels[3] = false;
+    }
+
+    // depth-only, make sure we display it as such
+    if(GetBaseFormat(texDetails.internalFormat) == eGL_DEPTH_COMPONENT)
+    {
+      channels[0] = true;
+      channels[1] = false;
+      channels[2] = false;
+      channels[3] = false;
+    }
+
+    if(!channels[0] && channels[1])
+    {
+      dsTexMode = eGL_STENCIL_INDEX;
+
+      // Stencil texture sampling is not normalized in OpenGL
+      intIdx = 1;
+      float rangeScale = 1.0f;
+      switch(texDetails.internalFormat)
+      {
+        case eGL_STENCIL_INDEX1: rangeScale = 1.0f; break;
+        case eGL_STENCIL_INDEX4: rangeScale = 16.0f; break;
+        default:
+          RDCWARN("Unexpected raw format for stencil visualization");
+        // fall through
+        case eGL_DEPTH24_STENCIL8:
+        case eGL_DEPTH32F_STENCIL8:
+        case eGL_DEPTH_STENCIL:
+        case eGL_STENCIL_INDEX8: rangeScale = 255.0f; break;
+        case eGL_STENCIL_INDEX16: rangeScale = 65535.0f; break;
+      }
+      minval *= rangeScale;
+      maxval *= rangeScale;
+    }
+    else
+    {
+      dsTexMode = eGL_DEPTH_COMPONENT;
+    }
+  }
+  else
+  {
+    if(details.format.compType == CompType::UInt)
+      intIdx = 1;
+    if(details.format.compType == CompType::SInt)
+      intIdx = 2;
+  }
+
  GL.glBindBufferBase(eGL_UNIFORM_BUFFER, 2, DebugData.UBOs[0]);
  HistogramUBOData *cdata =
      (HistogramUBOData *)GL.glMapBufferRange(eGL_UNIFORM_BUFFER, 0, sizeof(HistogramUBOData),
@@ -1156,28 +1292,30 @@ bool GLReplay::GetHistogram(ResourceId texid, uint32_t sliceFace, uint32_t mip,
  cdata->HistogramMax = maxval + maxval * 1e-6f;

  cdata->HistogramChannels = 0;
-  if(channels[0])
+  if(dsTexMode == eGL_NONE)
+  {
+    if(channels[0])
+      cdata->HistogramChannels |= 0x1;
+    if(channels[1])
+      cdata->HistogramChannels |= 0x2;
+    if(channels[2])
+      cdata->HistogramChannels |= 0x4;
+    if(channels[3])
+      cdata->HistogramChannels |= 0x8;
+  }
+  else
+  {
+    // Both depth and stencil texture mode use the red channel
    cdata->HistogramChannels |= 0x1;
-  if(channels[1])
-    cdata->HistogramChannels |= 0x2;
-  if(channels[2])
-    cdata->HistogramChannels |= 0x4;
-  if(channels[3])
-    cdata->HistogramChannels |= 0x8;
+  }
  cdata->HistogramFlags = 0;

  int progIdx = texSlot;

-  if(details.format.compType == CompType::UInt)
-  {
+  if(intIdx == 1)
    progIdx |= TEXDISPLAY_UINT_TEX;
-    intIdx = 1;
-  }
-  if(details.format.compType == CompType::SInt)
-  {
+  if(intIdx == 2)
    progIdx |= TEXDISPLAY_SINT_TEX;
-    intIdx = 2;
-  }

  int blocksX = (int)ceil(cdata->HistogramTextureResolution.x /
                          float(HGRAM_PIXELS_PER_TILE * HGRAM_TILES_PER_BLOCK));
@@ -1193,6 +1331,13 @@ bool GLReplay::GetHistogram(ResourceId texid, uint32_t sliceFace, uint32_t mip,
  else
    GL.glBindSampler(texSlot, DebugData.pointSampler);

+  GLint origDSTexMode = eGL_DEPTH_COMPONENT;
+  if(dsTexMode != eGL_NONE && HasExt[ARB_stencil_texturing])
+  {
+    GL.glGetTexParameteriv(target, eGL_DEPTH_STENCIL_TEXTURE_MODE, &origDSTexMode);
+    GL.glTexParameteri(target, eGL_DEPTH_STENCIL_TEXTURE_MODE, dsTexMode);
+  }
+
  int maxlevel = -1;

  int clampmaxlevel = details.mips - 1;
@@ -1229,6 +1374,9 @@ bool GLReplay::GetHistogram(ResourceId texid, uint32_t sliceFace, uint32_t mip,
  if(maxlevel >= 0)
    GL.glTextureParameterivEXT(texname, target, eGL_TEXTURE_MAX_LEVEL, (GLint *)&maxlevel);

+  if(dsTexMode != eGL_NONE && HasExt[ARB_stencil_texturing])
+    GL.glTexParameteri(target, eGL_DEPTH_STENCIL_TEXTURE_MODE, origDSTexMode);
+
  return true;
 }

@@ -244,6 +244,9 @@ private:
                            const rdcarray<ShaderConstant> &variables,
                            std::vector<ShaderVariable> &outvars, const bytebuf &data);

+  bool GetMinMax(ResourceId texid, uint32_t sliceFace, uint32_t mip, uint32_t sample,
+                 CompType typeHint, bool stencil, float *minval, float *maxval);
+
  void CreateCustomShaderTex(uint32_t w, uint32_t h);
  void CreateOverlayProgram(GLuint Program, GLuint Pipeline, GLuint fragShader);

@@ -1639,6 +1639,45 @@ void VulkanReplay::FillCBufferVariables(ResourceId shader, string entryPoint, ui

 bool VulkanReplay::GetMinMax(ResourceId texid, uint32_t sliceFace, uint32_t mip, uint32_t sample,
                             CompType typeHint, float *minval, float *maxval)
+{
+  ImageLayouts &layouts = m_pDriver->m_ImageLayouts[texid];
+
+  if(IsDepthAndStencilFormat(layouts.format))
+  {
+    // for depth/stencil we need to run the code twice - once to fetch depth and once to fetch
+    // stencil - since we can't process float depth and int stencil at the same time
+    Vec4f depth[2] = {
+        {0.0f, 0.0f, 0.0f, 0.0f}, {1.0f, 1.0f, 1.0f, 1.0f},
+    };
+    Vec4u stencil[2] = {{0, 0, 0, 0}, {1, 1, 1, 1}};
+
+    bool success =
+        GetMinMax(texid, sliceFace, mip, sample, typeHint, false, &depth[0].x, &depth[1].x);
+
+    if(!success)
+      return false;
+
+    success = GetMinMax(texid, sliceFace, mip, sample, typeHint, true, (float *)&stencil[0].x,
+                        (float *)&stencil[1].x);
+
+    if(!success)
+      return false;
+
+    // copy across into green channel, casting up to float, dividing by the range for this texture
+    depth[0].y = float(stencil[0].x) / 255.0f;
+    depth[1].y = float(stencil[1].x) / 255.0f;
+
+    memcpy(minval, &depth[0].x, sizeof(depth[0]));
+    memcpy(maxval, &depth[1].x, sizeof(depth[1]));
+
+    return true;
+  }
+
+  return GetMinMax(texid, sliceFace, mip, sample, typeHint, false, minval, maxval);
+}
+
+bool VulkanReplay::GetMinMax(ResourceId texid, uint32_t sliceFace, uint32_t mip, uint32_t sample,
+                             CompType typeHint, bool stencil, float *minval, float *maxval)
 {
  VkDevice dev = m_pDriver->GetDev();
  VkCommandBuffer cmd = m_pDriver->GetNextCmd();
@@ -1650,13 +1689,22 @@ bool VulkanReplay::GetMinMax(ResourceId texid, uint32_t sliceFace, uint32_t mip,

  VkImageAspectFlags aspectFlags = VK_IMAGE_ASPECT_COLOR_BIT;
  if(IsStencilOnlyFormat(layouts.format))
+  {
    aspectFlags = VK_IMAGE_ASPECT_STENCIL_BIT;
+  }
  else if(IsDepthOrStencilFormat(layouts.format))
+  {
    aspectFlags = VK_IMAGE_ASPECT_DEPTH_BIT;

+    // do a stencil min/max if stencil is selected
+    if(stencil)
+      aspectFlags = VK_IMAGE_ASPECT_STENCIL_BIT;
+  }
+
  CreateTexImageView(aspectFlags, liveIm, iminfo);

-  VkImageView liveImView = iminfo.view;
+  VkImageView liveImView =
+      (aspectFlags == VK_IMAGE_ASPECT_STENCIL_BIT ? iminfo.altViews[0] : iminfo.view);

  RDCASSERT(liveImView != VK_NULL_HANDLE);

@@ -1696,14 +1744,16 @@ bool VulkanReplay::GetMinMax(ResourceId texid, uint32_t sliceFace, uint32_t mip,
      textype = RESTYPE_TEX2DMS;
  }

+  if(aspectFlags == VK_IMAGE_ASPECT_STENCIL_BIT)
+  {
+    descSetBinding = 10;
+    intTypeIndex = 1;
+  }
+
  descSetBinding += textype;

  if(m_Histogram.m_MinMaxTilePipe[textype][intTypeIndex] == VK_NULL_HANDLE)
-  {
-    *minval = 0.0f;
-    *maxval = 1.0f;
    return false;
-  }

  VkDescriptorBufferInfo bufdescs[3];
  RDCEraseEl(bufdescs);
@@ -1927,10 +1977,18 @@ bool VulkanReplay::GetHistogram(ResourceId texid, uint32_t sliceFace, uint32_t m

  VkImageAspectFlags aspectFlags = VK_IMAGE_ASPECT_COLOR_BIT;
  if(IsStencilOnlyFormat(layouts.format))
+  {
    aspectFlags = VK_IMAGE_ASPECT_STENCIL_BIT;
+  }
  else if(IsDepthOrStencilFormat(layouts.format))
+  {
    aspectFlags = VK_IMAGE_ASPECT_DEPTH_BIT;

+    // detect if stencil is selected
+    if(!channels[0] && channels[1] && !channels[2] && !channels[3])
+      aspectFlags = VK_IMAGE_ASPECT_STENCIL_BIT;
+  }
+
  CreateTexImageView(aspectFlags, liveIm, iminfo);

  uint32_t descSetBinding = 0;
@@ -1964,6 +2022,19 @@ bool VulkanReplay::GetHistogram(ResourceId texid, uint32_t sliceFace, uint32_t m
      textype = RESTYPE_TEX2DMS;
  }

+  if(aspectFlags == VK_IMAGE_ASPECT_STENCIL_BIT)
+  {
+    descSetBinding = 10;
+    intTypeIndex = 1;
+
+    // rescale the range so that stencil seems to fit to 0-1
+    minval *= 255.0f;
+    maxval *= 255.0f;
+
+    // shuffle the channel selection, since stencil comes back in red
+    std::swap(channels[0], channels[1]);
+  }
+
  descSetBinding += textype;

  if(m_Histogram.m_HistogramPipe[textype][intTypeIndex] == VK_NULL_HANDLE)
@@ -349,6 +349,9 @@ private:

  bool RenderTextureInternal(TextureDisplay cfg, VkRenderPassBeginInfo rpbegin, int flags);

+  bool GetMinMax(ResourceId texid, uint32_t sliceFace, uint32_t mip, uint32_t sample,
+                 CompType typeHint, bool stencil, float *minval, float *maxval);
+
  void CreateTexImageView(VkImageAspectFlags aspectFlags, VkImage liveIm,
                          VulkanCreationInfo::Image &iminfo);