Split texture display shaders into precompiled variants

* GL won't let you define too many texture uniforms, and currently the tex
  sampling code blows it for the texture display shader. So we split it
  by float/uint/sint and pick the right compiled version.

renderdoc/data/glsl/debuguniforms.h

@@ -116,7 +116,6 @@ BINDING(0) uniform HistogramCBufferData
 #define TEXDISPLAY_TYPEMASK    0xF
 #define TEXDISPLAY_UINT_TEX    0x10
 #define TEXDISPLAY_SINT_TEX    0x20
-#define TEXDISPLAY_DEPTH_TEX   0x40
 #define TEXDISPLAY_NANS        0x80
 #define TEXDISPLAY_CLIPPING    0x100
 #define TEXDISPLAY_GAMMA_CURVE 0x200

renderdoc/data/glsl/texdisplay.frag

@@ -26,9 +26,16 @@ layout (location = 0) out vec4 color_out;
 
 void main(void)
 {
-	bool uintTex = (OutputDisplayFormat & TEXDISPLAY_UINT_TEX) != 0;
-	bool sintTex = (OutputDisplayFormat & TEXDISPLAY_SINT_TEX) != 0;
-	bool depthTex = (OutputDisplayFormat & TEXDISPLAY_DEPTH_TEX) != 0;
+#if UINT_TEX
+	bool uintTex = true;
+	bool sintTex = false;
+#elif SINT_TEX
+	bool uintTex = false;
+	bool sintTex = true;
+#else
+	bool uintTex = false;
+	bool sintTex = false;
+#endif
 
 	vec4 col;
 	uvec4 ucol;
@@ -55,18 +62,13 @@ void main(void)
 	}
 
 	// sample the texture.
-	if (uintTex)
-	{
+#if UINT_TEX
 		ucol = SampleTextureUInt4(scr, texType, FlipY == 0, int(MipLevel), Slice, SampleIdx);
-	}
-	else if (sintTex)
-	{
+#elif SINT_TEX
 		scol = SampleTextureSInt4(scr, texType, FlipY == 0, int(MipLevel), Slice, SampleIdx);
-	}
-	else
-	{
+#else
 		col = SampleTextureFloat4(scr, texType, FlipY == 0, int(MipLevel), Slice, SampleIdx, NumSamples);
-	}
+#endif
 
 	if(RawOutput != 0)
 	{

renderdoc/data/glsl/texsample.h

@@ -22,37 +22,6 @@
  * THE SOFTWARE.
  ******************************************************************************/
 
-// these bindings are defined based on the RESTYPE_ defines in debuguniforms.h
-// optionally TEXDISPLAY_UINT_TEX or TEXDISPLAY_SINT_TEX, OR'd with RESTYPE_*
-layout (binding = 1) uniform sampler1D tex1D;
-layout (binding = 2) uniform sampler2D tex2D;
-layout (binding = 3) uniform sampler3D tex3D;
-layout (binding = 4) uniform samplerCube texCube;
-layout (binding = 5) uniform sampler1DArray tex1DArray;
-layout (binding = 6) uniform sampler2DArray tex2DArray;
-layout (binding = 7) uniform samplerCubeArray texCubeArray;
-layout (binding = 8) uniform sampler2DRect tex2DRect;
-layout (binding = 9) uniform samplerBuffer texBuffer;
-layout (binding = 10) uniform sampler2DMS tex2DMS;
-
-layout (binding = 17) uniform usampler1D texUInt1D;
-layout (binding = 18) uniform usampler2D texUInt2D;
-layout (binding = 19) uniform usampler3D texUInt3D;
-layout (binding = 20) uniform usampler1DArray texUInt1DArray;
-layout (binding = 21) uniform usampler2DArray texUInt2DArray;
-layout (binding = 22) uniform usampler2DRect texUInt2DRect;
-layout (binding = 23) uniform usamplerBuffer texUIntBuffer;
-layout (binding = 24) uniform usampler2DMS texUInt2DMS;
-
-layout (binding = 33) uniform isampler1D texSInt1D;
-layout (binding = 34) uniform isampler2D texSInt2D;
-layout (binding = 35) uniform isampler3D texSInt3D;
-layout (binding = 36) uniform isampler1DArray texSInt1DArray;
-layout (binding = 37) uniform isampler2DArray texSInt2DArray;
-layout (binding = 38) uniform isampler2DRect texSInt2DRect;
-layout (binding = 39) uniform isamplerBuffer texSIntBuffer;
-layout (binding = 40) uniform isampler2DMS texSInt2DMS;
-
 vec3 CalcCubeCoord(vec2 uv, int face)
 {
 	// Map UVs to [-0.5, 0.5] and rotate
@@ -73,6 +42,21 @@ vec3 CalcCubeCoord(vec2 uv, int face)
 	return coord;
 }
 
+#if UINT_TEX
+
+// these bindings are defined based on the RESTYPE_ defines in debuguniforms.h
+
+layout (binding = 1) uniform usampler1D texUInt1D;
+layout (binding = 2) uniform usampler2D texUInt2D;
+layout (binding = 3) uniform usampler3D texUInt3D;
+// cube = 4
+layout (binding = 5) uniform usampler1DArray texUInt1DArray;
+layout (binding = 6) uniform usampler2DArray texUInt2DArray;
+// cube array = 7
+layout (binding = 8) uniform usampler2DRect texUInt2DRect;
+layout (binding = 9) uniform usamplerBuffer texUIntBuffer;
+layout (binding = 10) uniform usampler2DMS texUInt2DMS;
+
 uvec4 SampleTextureUInt4(vec2 pos, int type, bool flipY, int mipLevel, float slice, int sampleIdx)
 {
 	uvec4 col;
@@ -144,6 +128,21 @@ uvec4 SampleTextureUInt4(vec2 pos, int type, bool flipY, int mipLevel, float sli
 	return col;
 }
 
+#elif SINT_TEX
+
+// these bindings are defined based on the RESTYPE_ defines in debuguniforms.h
+
+layout (binding = 1) uniform isampler1D texSInt1D;
+layout (binding = 2) uniform isampler2D texSInt2D;
+layout (binding = 3) uniform isampler3D texSInt3D;
+// cube = 4
+layout (binding = 5) uniform isampler1DArray texSInt1DArray;
+layout (binding = 6) uniform isampler2DArray texSInt2DArray;
+// cube array = 7
+layout (binding = 8) uniform isampler2DRect texSInt2DRect;
+layout (binding = 9) uniform isamplerBuffer texSIntBuffer;
+layout (binding = 10) uniform isampler2DMS texSInt2DMS;
+
 ivec4 SampleTextureSInt4(vec2 pos, int type, bool flipY, int mipLevel, float slice, int sampleIdx)
 {
 	ivec4 col;
@@ -215,6 +214,20 @@ ivec4 SampleTextureSInt4(vec2 pos, int type, bool flipY, int mipLevel, float sli
 	return col;
 }
 
+#else
+
+// these bindings are defined based on the RESTYPE_ defines in debuguniforms.h
+
+layout (binding = 1) uniform sampler1D tex1D;
+layout (binding = 2) uniform sampler2D tex2D;
+layout (binding = 3) uniform sampler3D tex3D;
+layout (binding = 4) uniform samplerCube texCube;
+layout (binding = 5) uniform sampler1DArray tex1DArray;
+layout (binding = 6) uniform sampler2DArray tex2DArray;
+layout (binding = 7) uniform samplerCubeArray texCubeArray;
+layout (binding = 8) uniform sampler2DRect tex2DRect;
+layout (binding = 9) uniform samplerBuffer texBuffer;
+layout (binding = 10) uniform sampler2DMS tex2DMS;
 
 vec4 SampleTextureFloat4(vec2 pos, int type, bool flipY, int mipLevel, float slice, int sampleIdx, int sampleCount)
 {
@@ -357,3 +370,5 @@ vec4 SampleTextureFloat4(vec2 pos, int type, bool flipY, int mipLevel, float sli
 	
 	return col;
 }
+
+#endif

renderdoc/driver/gl/gl_debug.cpp

@@ -141,12 +141,21 @@ void GLReplay::InitDebugData()
 	DebugData.blitfsSource = GetEmbeddedResource(blit_frag);
 
 	DebugData.blitProg = CreateShaderProgram(DebugData.blitvsSource.c_str(), DebugData.blitfsSource.c_str());
+	
+	string glslheader = GetEmbeddedResource(debuguniforms_h);
 
-	string texfs = GetEmbeddedResource(debuguniforms_h);
-	texfs += GetEmbeddedResource(texsample_h);
+	string texfs = GetEmbeddedResource(texsample_h);
 	texfs += GetEmbeddedResource(texdisplay_frag);
 
-	DebugData.texDisplayProg = CreateShaderProgram(DebugData.blitvsSource.c_str(), texfs.c_str());
+	for(int i=0; i < 3; i++)
+	{
+		string glsl = glslheader;
+		glsl += string("#define UINT_TEX ") + (i == 1 ? "1" : "0") + "\n";
+		glsl += string("#define SINT_TEX ") + (i == 2 ? "1" : "0") + "\n";
+		glsl += texfs;
+
+		DebugData.texDisplayProg[i] = CreateShaderProgram(DebugData.blitvsSource.c_str(), glsl.c_str());
+	}
 
 	string checkerfs = GetEmbeddedResource(checkerboard_frag);
 	
@@ -194,6 +203,12 @@ void GLReplay::InitDebugData()
 	gl.glSamplerParameteri(DebugData.pointSampler, eGL_TEXTURE_WRAP_S, eGL_CLAMP_TO_EDGE);
 	gl.glSamplerParameteri(DebugData.pointSampler, eGL_TEXTURE_WRAP_T, eGL_CLAMP_TO_EDGE);
 	
+	gl.glGenSamplers(1, &DebugData.pointNoMipSampler);
+	gl.glSamplerParameteri(DebugData.pointNoMipSampler, eGL_TEXTURE_MIN_FILTER, eGL_NEAREST);
+	gl.glSamplerParameteri(DebugData.pointNoMipSampler, eGL_TEXTURE_MAG_FILTER, eGL_NEAREST);
+	gl.glSamplerParameteri(DebugData.pointNoMipSampler, eGL_TEXTURE_WRAP_S, eGL_CLAMP_TO_EDGE);
+	gl.glSamplerParameteri(DebugData.pointNoMipSampler, eGL_TEXTURE_WRAP_T, eGL_CLAMP_TO_EDGE);
+	
 	gl.glGenBuffers(ARRAY_COUNT(DebugData.UBOs), DebugData.UBOs);
 	for(size_t i=0; i < ARRAY_COUNT(DebugData.UBOs); i++)
 	{
@@ -225,8 +240,6 @@ void GLReplay::InitDebugData()
 	
 	// histogram/minmax data
 	{
-		string glslheader = GetEmbeddedResource(debuguniforms_h);
-
 		string histogramglsl = GetEmbeddedResource(texsample_h);
 		histogramglsl += GetEmbeddedResource(histogram_comp);
 
@@ -407,14 +420,16 @@ bool GLReplay::GetMinMax(ResourceId texid, uint32_t sliceFace, uint32_t mip, uin
 	cdata->HistogramMax = 1.0f;
 	cdata->HistogramChannels = 0xf;
 	
+	int progIdx = texSlot;
+
 	if(details.format.compType == eCompType_UInt)
 	{
-		texSlot |= TEXDISPLAY_UINT_TEX;
+		progIdx |= TEXDISPLAY_UINT_TEX;
 		intIdx = 1;
 	}
 	if(details.format.compType == eCompType_SInt)
 	{
-		texSlot |= TEXDISPLAY_SINT_TEX;
+		progIdx |= TEXDISPLAY_SINT_TEX;
 		intIdx = 2;
 	}
 	
@@ -428,11 +443,14 @@ bool GLReplay::GetMinMax(ResourceId texid, uint32_t sliceFace, uint32_t mip, uin
 
 	gl.glActiveTexture((RDCGLenum)(eGL_TEXTURE0 + texSlot));
 	gl.glBindTexture(target, texname);
-	gl.glBindSampler(texSlot, DebugData.pointSampler);
+	if(texSlot == RESTYPE_TEXRECT || texSlot == RESTYPE_TEXBUFFER)
+		gl.glBindSampler(texSlot, DebugData.pointNoMipSampler);
+	else
+		gl.glBindSampler(texSlot, DebugData.pointSampler);
 	
 	gl.glBindBufferBase(eGL_SHADER_STORAGE_BUFFER, 0, DebugData.minmaxTileResult);
 
-	gl.glUseProgram(DebugData.minmaxTileProgram[texSlot]);
+	gl.glUseProgram(DebugData.minmaxTileProgram[progIdx]);
 	gl.glDispatchCompute(blocksX, blocksY, 1);
 
 	gl.glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT);
@@ -572,14 +590,16 @@ bool GLReplay::GetHistogram(ResourceId texid, uint32_t sliceFace, uint32_t mip,
 	if(channels[3]) cdata->HistogramChannels |= 0x8;
 	cdata->HistogramFlags = 0;
 
+	int progIdx = texSlot;
+
 	if(details.format.compType == eCompType_UInt)
 	{
-		texSlot |= TEXDISPLAY_UINT_TEX;
+		progIdx |= TEXDISPLAY_UINT_TEX;
 		intIdx = 1;
 	}
 	if(details.format.compType == eCompType_SInt)
 	{
-		texSlot |= TEXDISPLAY_SINT_TEX;
+		progIdx |= TEXDISPLAY_SINT_TEX;
 		intIdx = 2;
 	}
 
@@ -593,14 +613,17 @@ bool GLReplay::GetHistogram(ResourceId texid, uint32_t sliceFace, uint32_t mip,
 
 	gl.glActiveTexture((RDCGLenum)(eGL_TEXTURE0 + texSlot));
 	gl.glBindTexture(target, texname);
-	gl.glBindSampler(texSlot, DebugData.pointSampler);
+	if(texSlot == RESTYPE_TEXRECT || texSlot == RESTYPE_TEXBUFFER)
+		gl.glBindSampler(texSlot, DebugData.pointNoMipSampler);
+	else
+		gl.glBindSampler(texSlot, DebugData.pointSampler);
 
 	gl.glBindBufferBase(eGL_SHADER_STORAGE_BUFFER, 0, DebugData.histogramBuf);
 
 	GLuint zero = 0;
 	gl.glClearBufferData(eGL_SHADER_STORAGE_BUFFER, eGL_R32UI, eGL_RED, eGL_UNSIGNED_INT, &zero);
 
-	gl.glUseProgram(DebugData.histogramProgram[texSlot]);
+	gl.glUseProgram(DebugData.histogramProgram[progIdx]);
 	gl.glDispatchCompute(blocksX, blocksY, 1);
 
 	gl.glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT);
@@ -662,6 +685,8 @@ bool GLReplay::RenderTexture(TextureDisplay cfg)
 
 	bool renderbuffer = false;
 
+	int intIdx = 0;
+
 	int resType;
 	switch (texDetails.curType)
 	{
@@ -734,8 +759,6 @@ bool GLReplay::RenderTexture(TextureDisplay cfg)
 	
 	MakeCurrentReplayContext(m_DebugCtx);
 	
-	gl.glUseProgram(DebugData.texDisplayProg);
-
 	RDCGLenum dsTexMode = eGL_NONE;
 	if(IsDepthStencilFormat(texDetails.internalFormat))
 	{
@@ -744,7 +767,7 @@ bool GLReplay::RenderTexture(TextureDisplay cfg)
 			dsTexMode = eGL_STENCIL_INDEX;
 
 			// Stencil texture sampling is not normalized in OpenGL
-			resType |= TEXDISPLAY_UINT_TEX;
+			intIdx = 1;
 			float rangeScale;
 			switch (texDetails.internalFormat)
 			{
@@ -774,11 +797,13 @@ bool GLReplay::RenderTexture(TextureDisplay cfg)
 	else
 	{
 		if(IsUIntFormat(texDetails.internalFormat))
-				resType |= TEXDISPLAY_UINT_TEX;
+				intIdx = 1;
 		if(IsSIntFormat(texDetails.internalFormat))
-				resType |= TEXDISPLAY_SINT_TEX;
+				intIdx = 2;
 	}
 	
+	gl.glUseProgram(DebugData.texDisplayProg[intIdx]);
+
 	gl.glActiveTexture((RDCGLenum)(eGL_TEXTURE0 + resType));
 	gl.glBindTexture(target, texname);
 
@@ -805,13 +830,16 @@ bool GLReplay::RenderTexture(TextureDisplay cfg)
 		maxlevel = -1;
 	}
 
-	if(cfg.mip == 0 && cfg.scale < 1.0f && dsTexMode == eGL_NONE && resType != eGL_TEXTURE_BUFFER)
+	if(cfg.mip == 0 && cfg.scale < 1.0f && dsTexMode == eGL_NONE && resType != RESTYPE_TEXBUFFER && resType != RESTYPE_TEXRECT)
 	{
 		gl.glBindSampler(resType, DebugData.linearSampler);
 	}
 	else
 	{
-		gl.glBindSampler(resType, DebugData.pointSampler);
+		if(resType == RESTYPE_TEXRECT || resType == RESTYPE_TEX2DMS || resType == RESTYPE_TEXBUFFER)
+			gl.glBindSampler(resType, DebugData.pointNoMipSampler);
+		else
+			gl.glBindSampler(resType, DebugData.pointSampler);
 	}
 	
 	GLint tex_x = texDetails.width, tex_y = texDetails.height, tex_z = texDetails.depth;
@@ -875,9 +903,6 @@ bool GLReplay::RenderTexture(TextureDisplay cfg)
 	ubo->Slice = (float)cfg.sliceFace;
 
 	ubo->OutputDisplayFormat = resType;
-
-	if(dsTexMode != eGL_NONE)
-		ubo->OutputDisplayFormat |= TEXDISPLAY_DEPTH_TEX;
 	
 	if(cfg.overlay == eTexOverlay_NaN)
 		ubo->OutputDisplayFormat |= TEXDISPLAY_NANS;

renderdoc/driver/gl/gl_replay.h

@@ -189,9 +189,10 @@ class GLReplay : public IReplayDriver
 			// no transformation or scaling
 			GLuint blitProg;
 
-			GLuint texDisplayProg;
+			GLuint texDisplayProg[3];      // float/uint/sint
 
 			GLuint pointSampler;
+			GLuint pointNoMipSampler;
 			GLuint linearSampler;
 
 			GLuint checkerProg;