From 9856b171c10cb19582008dcd3ccdc06359003141 Mon Sep 17 00:00:00 2001 From: baldurk Date: Mon, 26 Oct 2015 18:15:31 +0100 Subject: [PATCH] Take stencil value from max of red or green channel * I swear some implementations return stencil in the green channel, but the spec says red channel. This should be safe --- renderdoc/driver/gl/gl_debug.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/renderdoc/driver/gl/gl_debug.cpp b/renderdoc/driver/gl/gl_debug.cpp index 98ca5f7d8..e5a278784 100644 --- a/renderdoc/driver/gl/gl_debug.cpp +++ b/renderdoc/driver/gl/gl_debug.cpp @@ -1241,13 +1241,15 @@ void GLReplay::PickPixel(ResourceId texture, uint32_t x, uint32_t y, uint32_t sl uint32_t stencilpixel[4]; gl.glReadPixels(0, 0, 1, 1, eGL_RGBA, eGL_FLOAT, (void *)stencilpixel); - pixel[1] = float(stencilpixel[1])/255.0f; + // not sure whether [0] or [1] will return stencil values, so use + // max of two because other channel should be 0 + pixel[1] = float(RDCMAX(stencilpixel[0], stencilpixel[1]))/255.0f; // the first depth read will have read stencil instead. // NULL it out so the UI sees only stencil if(texDetails.internalFormat == eGL_STENCIL_INDEX8) { - pixel[1] = float(stencilpixel[0])/255.0f; + pixel[1] = float(RDCMAX(stencilpixel[0], stencilpixel[1]))/255.0f; pixel[0] = 0.0f; } }