Clean up D3D11 shader debugging test for easier processing, auto-test it

util/test/demos/d3d11/d3d11_shader_debug_zoo.cpp

@@ -24,11 +24,9 @@
 
 #include "d3d11_test.h"
 
-TEST(D3D11_Debug_Shader, D3D11GraphicsTest)
+TEST(D3D11_Shader_Debug_Zoo, D3D11GraphicsTest)
 {
-  static constexpr const char *Description =
-      "Tests simple shader debugging identities by rendering many small triangles and "
-      "performing one calculation to each to an F32 target";
+  static constexpr const char *Description = "Tests shader debugging in different edge cases";
 
   struct ConstsA2V
   {
@@ -65,13 +63,7 @@ v2f main(consts IN, uint tri : SV_InstanceID)
 {
 	v2f OUT = (v2f)0;
 
-	OUT.pos = float4(IN.pos.xyz * 0.1f, 1);
-
-	uint row = tri / 10;
-	uint col = tri % 10;
-
-	OUT.pos.x += -0.9f + col*0.2f;
-	OUT.pos.y +=  0.85f - row*0.3f;
+	OUT.pos = float4(IN.pos.x + IN.pos.z * float(tri), IN.pos.y, 0.0f, 1);
 
 	OUT.zeroVal = IN.zeroVal.xx;
 	OUT.oneVal = IN.oneVal;
@@ -203,6 +195,8 @@ float4 main(v2f IN) : SV_Target0
 
 )EOSHADER";
 
+  static const uint32_t numTests = 34;
+
   int main()
   {
     // initialise, create window, create device, etc
@@ -237,14 +231,17 @@ float4 main(v2f IN) : SV_Target0
     ID3D11VertexShaderPtr vs = CreateVS(vsblob);
     ID3D11PixelShaderPtr ps = CreatePS(psblob);
 
-    ID3D11Texture2DPtr fltTex =
-        MakeTexture(DXGI_FORMAT_R32G32B32A32_FLOAT, screenWidth, screenHeight).RTV();
+    static const uint32_t texDim = AlignUp(numTests, 64U) * 4;
+
+    ID3D11Texture2DPtr fltTex = MakeTexture(DXGI_FORMAT_R32G32B32A32_FLOAT, texDim, 4).RTV();
     ID3D11RenderTargetViewPtr fltRT = MakeRTV(fltTex);
 
+    float triWidth = 8.0f / float(texDim);
+
     ConstsA2V triangle[] = {
-        {Vec3f(-0.5f, 0.0f, 0.0f), 0.0f, 1.0f, -1.0f},
-        {Vec3f(0.0f, 1.0f, 0.0f), 0.0f, 1.0f, -1.0f},
-        {Vec3f(0.5f, 0.0f, 0.0f), 0.0f, 1.0f, -1.0f},
+        {Vec3f(-1.0f, -1.0f, triWidth), 0.0f, 1.0f, -1.0f},
+        {Vec3f(-1.0f, 1.0f, triWidth), 0.0f, 1.0f, -1.0f},
+        {Vec3f(-1.0f + triWidth, 1.0f, triWidth), 0.0f, 1.0f, -1.0f},
     };
 
     ID3D11BufferPtr vb = MakeBuffer().Vertex().Data(triangle);
@@ -271,11 +268,11 @@ float4 main(v2f IN) : SV_Target0
       ctx->VSSetShader(vs, NULL, 0);
       ctx->PSSetShader(ps, NULL, 0);
 
-      RSSetViewport({0.0f, 0.0f, (float)screenWidth, (float)screenHeight, 0.0f, 1.0f});
+      RSSetViewport({0.0f, 0.0f, (float)texDim, 4.0f, 0.0f, 1.0f});
 
       ctx->OMSetRenderTargets(1, &fltRT.GetInterfacePtr(), NULL);
 
-      ctx->DrawInstanced(3, 70, 0, 0);
+      ctx->DrawInstanced(3, numTests, 0, 0);
 
       Present();
     }

util/test/demos/demos.vcxproj

@@ -137,7 +137,7 @@
     <ClCompile Include="d3d11\d3d11_mip_gen_rt.cpp" />
     <ClCompile Include="d3d11\d3d11_mip_rtv.cpp" />
     <ClCompile Include="d3d11\d3d11_overdraw_stress.cpp" />
-    <ClCompile Include="d3d11\d3d11_debug_shader.cpp" />
+    <ClCompile Include="d3d11\d3d11_shader_debug_zoo.cpp" />
     <ClCompile Include="d3d11\d3d11_overlay_test.cpp" />
     <ClCompile Include="d3d11\d3d11_primitive_restart.cpp" />
     <ClCompile Include="d3d11\d3d11_refcount_check.cpp" />

util/test/demos/demos.vcxproj.filters

@@ -16,9 +16,6 @@
     <ClCompile Include="d3d11\d3d11_1_many_uavs.cpp">
       <Filter>D3D11\demos</Filter>
     </ClCompile>
-    <ClCompile Include="d3d11\d3d11_debug_shader.cpp">
-      <Filter>D3D11\demos</Filter>
-    </ClCompile>
     <ClCompile Include="d3d11\d3d11_mip_rtv.cpp">
       <Filter>D3D11\demos</Filter>
     </ClCompile>
@@ -300,6 +297,9 @@
     <ClCompile Include="gl\gl_per_type_tex_units.cpp">
       <Filter>OpenGL\demos</Filter>
     </ClCompile>
+    <ClCompile Include="d3d11\d3d11_shader_debug_zoo.cpp">
+      <Filter>D3D11\demos</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <Filter Include="D3D11">

util/test/rdtest/util.py

@@ -2,6 +2,7 @@ import sys
 import os
 import re
 import time
+import math
 import struct
 import platform
 import hashlib
@@ -244,6 +245,10 @@ def value_compare(ref, data):
         if type(data) != float:
             return False
 
+        # Special handling for NaNs
+        if math.isnan(ref) == math.isnan(data):
+            return True
+
         # Floats are equal if the absolute difference is less than epsilon times the largest.
         largest = max(abs(ref), abs(data))
         eps = largest * FLT_EPSILON if largest > 1.0 else FLT_EPSILON

util/test/tests/D3D11/D3D11_Shader_Debug_Zoo.py

@@ -0,0 +1,41 @@
+import renderdoc as rd
+import rdtest
+
+
+class D3D11_Shader_Debug_Zoo(rdtest.TestCase):
+    demos_test_name = 'D3D11_Shader_Debug_Zoo'
+
+    def check_capture(self):
+        # Jump to the draw
+        draw = self.find_draw("Draw")
+
+        self.controller.SetFrameEvent(draw.eventId, False)
+
+        # Make an output so we can pick pixels
+        out: rd.ReplayOutput = self.controller.CreateOutput(rd.CreateHeadlessWindowingData(100, 100), rd.ReplayOutputType.Texture)
+
+        self.check(out is not None)
+
+        pipe: rd.PipeState = self.controller.GetPipelineState()
+
+        tex = rd.TextureDisplay()
+        tex.resourceId = pipe.GetOutputTargets()[0].resourceId
+        out.SetTextureDisplay(tex)
+
+        # Loop over every test
+        for test in range(draw.numInstances):
+            # Pick the pixel
+            picked: rd.PixelValue = out.PickPixel(tex.resourceId, False, 4 * test, 0, 0, 0, 0)
+
+            # Debug the shader
+            trace: rd.ShaderDebugTrace = self.controller.DebugPixel(4 * test, 0, rd.ReplayController.NoPreference,
+                                                                    rd.ReplayController.NoPreference)
+
+            last_state: rd.ShaderDebugState = trace.states[-1]
+
+            if not rdtest.value_compare(picked.floatValue, last_state.outputs[0].value.fv[0:4]):
+                raise rdtest.TestFailureException("Test {}: debugged output {} doesn't match actual output {}".format(test, last_state.outputs[0].value.fv[0:4], picked.floatValue))
+
+            rdtest.log.success("Test {} matched as expected".format(test))
+
+        rdtest.log.success("All tests matched")