Implement NaN/inf/-ve and clipping overlays on OpenGL

renderdoc/data/glsl/debuguniforms.h

@@ -109,7 +109,12 @@ BINDING(0) uniform HistogramCBufferData
 #define TEXDISPLAY_UINT_TEX   0x8
 #define TEXDISPLAY_SINT_TEX   0x10
 #define TEXDISPLAY_DEPTH_TEX  0x20
+#define TEXDISPLAY_NANS       0x40
+#define TEXDISPLAY_CLIPPING   0x80
 
+#ifndef FLT_EPSILON
+#define FLT_EPSILON 1.192092896e-07f
+#endif
 
 // histogram/minmax is calculated in blocks of NxN each with MxM tiles.
 // e.g. a tile is 32x32 pixels, then this is arranged in blocks of 32x32 tiles.

renderdoc/data/glsl/texdisplay.frag

@@ -98,9 +98,47 @@ void main(void)
 	col = ((col - RangeMinimum)*InverseRangeSize);
 
 	col = mix(vec4(0,0,0,1), col, Channels);
+	
+	// show nans, infs and negatives
+	if((OutputDisplayFormat & TEXDISPLAY_NANS) > 0)
+	{
+		if(isnan(col.r) || isnan(col.g) || isnan(col.b) || isnan(col.a))
+		{
+		   color_out = vec4(1, 0, 0, 1);
+		   return;
+		}
+		   
+		if(isinf(col.r) || isinf(col.g) || isinf(col.b) || isinf(col.a))
+		{
+		   color_out = vec4(0, 1, 0, 1);
+		   return;
+		}
 
-	// TODO: check OutputDisplayFormat to see if we should highlight NaNs or clipping
-	// else
+		if(col.r < 0 || col.g < 0 || col.b < 0 || col.a < 0)
+		{
+		   color_out = vec4(0, 0, 1, 1);
+		   return;
+		}
+		
+		col = vec4(dot(col.xyz, vec3(0.2126, 0.7152, 0.0722)).xxx, 1);
+	}
+	else if((OutputDisplayFormat & TEXDISPLAY_CLIPPING) > 0)
+	{
+		if(col.r < 0 || col.g < 0 || col.b < 0 || col.a < 0)
+		{
+		   color_out = vec4(1, 0, 0, 1);
+		   return;
+		}
+
+		if(col.r > (1+FLT_EPSILON) || col.g > (1+FLT_EPSILON) || col.b > (1+FLT_EPSILON) || col.a > (1+FLT_EPSILON))
+		{
+		   color_out = vec4(0, 1, 0, 1);
+		   return;
+		}
+		
+		col = vec4(dot(col.xyz, vec3(0.2126, 0.7152, 0.0722)).xxx, 1);
+	}
+	else
 	{
 		// if only one channel is selected
 		if(dot(Channels, 1.0f.xxxx) == 1.0f.xxxx)

renderdoc/driver/gl/gl_debug.cpp

@@ -725,6 +725,12 @@ bool GLReplay::RenderTexture(TextureDisplay cfg)
 
 	if(dsTexMode != eGL_NONE)
 		ubo->OutputDisplayFormat |= TEXDISPLAY_DEPTH_TEX;
+	
+	if(cfg.overlay == eTexOverlay_NaN)
+		ubo->OutputDisplayFormat |= TEXDISPLAY_NANS;
+
+	if(cfg.overlay == eTexOverlay_Clipping)
+		ubo->OutputDisplayFormat |= TEXDISPLAY_CLIPPING;
 
 	ubo->RawOutput = cfg.rawoutput ? 1 : 0;