From d0166c0984500cfb4a8bde0ce7772f01749bf6e5 Mon Sep 17 00:00:00 2001
From: Jake Turner <jake@evansturner.co.uk>
Date: Wed, 10 Jan 2024 12:12:05 +0000
Subject: [PATCH] GL Depth Overlay use non-stencil mask approach by default

Only use stencil mask approach for depth exporting shaders.
The stencil mask approach does not show discarded pixels as passing.
---
 renderdoc/driver/gl/gl_overlay.cpp | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/renderdoc/driver/gl/gl_overlay.cpp b/renderdoc/driver/gl/gl_overlay.cpp
index beb2ff82c..adda408bb 100644
--- a/renderdoc/driver/gl/gl_overlay.cpp
+++ b/renderdoc/driver/gl/gl_overlay.cpp
@@ -1011,9 +1011,24 @@ ResourceId GLReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, Debug
     GLenum copyQueryEnum = texQueryEnum;
 
     GLuint depthCopy = 0, stencilCopy = 0;
-    bool useBlitFramebuffer = true;
     bool useDepthStencilMask = (overlay == DebugOverlay::Depth) && (curDepth != 0);
 
+    if(useDepthStencilMask)
+    {
+      useDepthStencilMask = false;
+      PerStageReflections stages;
+      m_pDriver->FillReflectionArray(rs.Program, stages);
+      const ShaderReflection *reflection = stages.refls[(uint32_t)ShaderStage::Fragment];
+      if(reflection)
+      {
+        for(const SigParameter &output : reflection->outputSignature)
+        {
+          if(output.systemValue == ShaderBuiltin::DepthOutput)
+            useDepthStencilMask = true;
+        }
+      }
+    }
+
     // create matching depth for existing FBO
     if(curDepth != 0)
     {
@@ -1051,8 +1066,8 @@ ResourceId GLReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, Debug
         else if(depth == 0 && stencil == 8)
           fmt = eGL_STENCIL_INDEX8;
       }
-      // For depth overlay : need a stencil buffer
-      if(overlay == DebugOverlay::Depth)
+      // Need a stencil buffer for the depth overlay stencil mask method
+      if(useDepthStencilMask)
       {
         GLenum oldFmt = fmt;
         if((oldFmt == eGL_DEPTH_COMPONENT16) || (oldFmt == eGL_DEPTH_COMPONENT24) ||
@@ -1066,11 +1081,11 @@ ResourceId GLReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, Debug
           if(DebugData.overlayTexSlices > 1)
           {
             useDepthStencilMask = false;
+            fmt = oldFmt;
             RDCWARN("Depth overlay using fallback method instead of stencil mask");
           }
           else
           {
-            useBlitFramebuffer = false;
             curStencil = curDepth;
           }
         }
@@ -1271,6 +1286,7 @@ ResourceId GLReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, Debug
                                  stencilCopy, sub.mip);
     }
 
+    bool useBlitFramebuffer = !useDepthStencilMask;
     if(useBlitFramebuffer)
     {
       // get latest depth/stencil from read FBO (existing FBO) into draw FBO (overlay FBO)