Use demo project name to determine test availability

* This is better than testing by platform/platform version, because we can check
  specifically for extensions and avoid running a test we know will fail because
  extensions aren't supported.

util/test/demos/d3d11/d3d11_primitive_restart.cpp

@@ -24,7 +24,7 @@
 
 #include "d3d11_test.h"
 
-struct Primitive_Restart : D3D11GraphicsTest
+struct D3D11_Primitive_Restart : D3D11GraphicsTest
 {
   static constexpr const char *Description =
       "Test of primitive restart in triangle strips with -1 index";
@@ -123,4 +123,4 @@ struct Primitive_Restart : D3D11GraphicsTest
   }
 };
 
-REGISTER_TEST(Primitive_Restart);
+REGISTER_TEST(D3D11_Primitive_Restart);

util/test/demos/gl/gl_dx_interop.cpp

@@ -29,7 +29,7 @@
 
 #include "3rdparty/glad/glad_wgl.h"
 
-struct DX_Interop : OpenGLGraphicsTest
+struct GL_DX_Interop : OpenGLGraphicsTest
 {
   static constexpr const char *Description =
       "Test interop between GL and DX (Create and render to a DX surface and include into "
@@ -321,6 +321,6 @@ void main()
   }
 };
 
-REGISTER_TEST(DX_Interop);
+REGISTER_TEST(GL_DX_Interop);
 
 #endif

util/test/demos/gl/gl_vao_0.cpp

@@ -24,7 +24,7 @@
 
 #include "gl_test.h"
 
-struct VAO_0 : OpenGLGraphicsTest
+struct GL_VAO_0 : OpenGLGraphicsTest
 {
   static constexpr const char *Description = "Uses VAO 0 (i.e. never binds a VAO)";
 
@@ -151,4 +151,4 @@ void main()
   }
 };
 
-REGISTER_TEST(VAO_0);
+REGISTER_TEST(GL_VAO_0);

util/test/demos/vk/vk_secondary_cmdbuf.cpp

@@ -24,7 +24,7 @@
 
 #include "vk_test.h"
 
-struct Secondary_CmdBuf : VulkanGraphicsTest
+struct VK_Secondary_CmdBuf : VulkanGraphicsTest
 {
   static constexpr const char *Description = "Draw using secondary command buffers";
 
@@ -238,4 +238,4 @@ void main()
   }
 };
 
-REGISTER_TEST(Secondary_CmdBuf);
+REGISTER_TEST(VK_Secondary_CmdBuf);

util/test/demos/vk/vk_vs_max_desc_set.cpp

@@ -24,7 +24,7 @@
 
 #include "vk_test.h"
 
-struct VS_Max_Desc_Set : VulkanGraphicsTest
+struct VK_VS_Max_Desc_Set : VulkanGraphicsTest
 {
   static constexpr const char *Description =
       "Uses the device's maximum number of descriptor sets in the vertex shader.";
@@ -197,4 +197,4 @@ void main()
   }
 };
 
-REGISTER_TEST(VS_Max_Desc_Set);
+REGISTER_TEST(VK_VS_Max_Desc_Set);

util/test/rdtest/runner.py

@@ -141,6 +141,18 @@ def _run_test(testclass, failedcases: list):
                            .format(test_run.returncode))
 
 
+def fetch_tests():
+    output = subprocess.run([util.get_demos_binary(), '--list-raw'], stdout=subprocess.PIPE).stdout
+
+    # Skip the header, grab all the remaining lines
+    tests = str(output, 'utf-8').splitlines()[1:]
+
+    # Split the TSV values and store
+    split_tests = [ test.split('\t') for test in tests ]
+
+    return { x[0]: (x[1] == 'True', x[2]) for x in split_tests }
+
+
 def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests: bool, debugger: bool):
     start_time = time.time()
 
@@ -228,6 +240,8 @@ def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests
 
     os.environ['RENDERDOC_DEMOS_DATA'] = util.get_data_path('demos')
 
+    testcase.TestCase.set_test_list(fetch_tests())
+
     testcases = get_tests()
 
     include_regexp = re.compile(test_include, re.IGNORECASE)
@@ -254,9 +268,12 @@ def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests
     for testclass in testcases:
         name = testclass.__name__
 
-        if ((testclass.platform != '' and testclass.platform != plat) or
-                (testclass.platform_version != 0 and testclass.platform_version > ver)):
-            log.print("Skipping {} as it's not supported on this platform '{} version {}'".format(name, plat, ver))
+        instance = testclass()
+
+        supported,unsupported_reason = instance.check_support()
+
+        if not supported:
+            log.print("Skipping {} as {}".format(name, unsupported_reason))
             skippedcases.append(testclass)
             continue
 
@@ -282,7 +299,6 @@ def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests
 
         def do():
             if in_process:
-                instance = testclass()
                 instance.invoketest()
             else:
                 _run_test(testclass, failedcases)

util/test/rdtest/testcase.py

@@ -5,6 +5,7 @@ import re
 import renderdoc as rd
 from . import util
 from . import analyse
+from . import capture
 from .logging import log, TestFailureException
 
 
@@ -126,8 +127,22 @@ class ConstantBufferChecker:
 
 class TestCase:
     slow_test = False
-    platform = ''
-    platform_version = 0
+    demos_test_name = ''
+    demos_frame_cap = 5
+    _test_list = {}
+
+    @staticmethod
+    def set_test_list(tests):
+        TestCase._test_list = tests
+
+    def check_support(self):
+        if self.demos_test_name != '':
+            if self.demos_test_name not in TestCase._test_list:
+                return False,'Test {} not in compiled tests'.format(self.demos_test_name)
+            return TestCase._test_list[self.demos_test_name]
+
+        # Otherwise assume we can run - child tests can override if they want to do some other check
+        return True,""
 
     def __init__(self):
         self.capture_filename = ""
@@ -161,6 +176,10 @@ class TestCase:
         :return: The path to the capture to open. If in a temporary path, it will be
           deleted if the test completes.
         """
+
+        if self.demos_test_name != '':
+            return capture.run_and_capture(util.get_demos_binary(), self.demos_test_name, self.demos_frame_cap)
+
         raise NotImplementedError("If run() is not implemented in a test, then"
                                   "get_capture() and check_capture() must be.")
 

util/test/tests/D3D11/D3D11_CBuffer_Zoo.py

@@ -3,10 +3,7 @@ import renderdoc as rd
 
 
 class D3D11_CBuffer_Zoo(rdtest.TestCase):
-    platform = 'win32'
-
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "D3D11_CBuffer_Zoo", 5)
+    demos_test_name = 'D3D11_CBuffer_Zoo'
 
     def check_capture(self):
         draw = self.find_draw("Draw")

util/test/tests/D3D11/D3D11_Overlay_Test.py

@@ -3,10 +3,7 @@ import renderdoc as rd
 
 
 class D3D11_Overlay_Test(rdtest.TestCase):
-    platform = 'win32'
-
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "D3D11_Overlay_Test", 5)
+    demos_test_name = 'D3D11_Overlay_Test'
 
     def check_capture(self):
         self.check_final_backbuffer()

util/test/tests/D3D11/D3D11_Primitive_Restart.py

@@ -3,10 +3,7 @@ import renderdoc as rd
 
 
 class D3D11_Primitive_Restart(rdtest.TestCase):
-    platform = 'win32'
-
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "Primitive_Restart", 5)
+    demos_test_name = 'D3D11_Primitive_Restart'
 
     def check_capture(self):
         self.check_final_backbuffer()

util/test/tests/D3D11/D3D11_Resource_Lifetimes.py

@@ -3,10 +3,7 @@ import rdtest
 
 
 class D3D11_Resource_Lifetimes(rdtest.TestCase):
-    platform = 'win32'
-
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "D3D11_Resource_Lifetimes", 5)
+    demos_test_name = 'D3D11_Resource_Lifetimes'
 
     def check_capture(self):
         self.check_final_backbuffer()

util/test/tests/D3D11/D3D11_Simple_Triangle.py

@@ -3,10 +3,7 @@ import rdtest
 
 
 class D3D11_Simple_Triangle(rdtest.TestCase):
-    platform = 'win32'
-
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "D3D11_Simple_Triangle", 5)
+    demos_test_name = 'D3D11_Simple_Triangle'
 
     def check_capture(self):
         self.check_final_backbuffer()

util/test/tests/D3D12/D3D12_CBuffer_Zoo.py

@@ -3,11 +3,7 @@ import renderdoc as rd
 
 
 class D3D12_CBuffer_Zoo(rdtest.TestCase):
-    platform = 'win32'
-    platform_version = 10
-
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "D3D12_CBuffer_Zoo", 5)
+    demos_test_name = 'D3D12_CBuffer_Zoo'
 
     def check_capture(self):
         draw = self.find_draw("Draw")

util/test/tests/D3D12/D3D12_Overlay_Test.py

@@ -3,11 +3,7 @@ import renderdoc as rd
 
 
 class D3D12_Overlay_Test(rdtest.TestCase):
-    platform = 'win32'
-    platform_version = 10
-
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "D3D12_Overlay_Test", 5)
+    demos_test_name = 'D3D12_Overlay_Test'
 
     def check_capture(self):
         self.check_final_backbuffer()

util/test/tests/D3D12/D3D12_Resource_Lifetimes.py

@@ -3,11 +3,7 @@ import rdtest
 
 
 class D3D12_Resource_Lifetimes(rdtest.TestCase):
-    platform = 'win32'
-    platform_version = 10
-
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "D3D12_Resource_Lifetimes", 5)
+    demos_test_name = 'D3D12_Resource_Lifetimes'
 
     def check_capture(self):
         self.check_final_backbuffer()

util/test/tests/D3D12/D3D12_Simple_Triangle.py

@@ -3,11 +3,7 @@ import rdtest
 
 
 class D3D12_Simple_Triangle(rdtest.TestCase):
-    platform = 'win32'
-    platform_version = 10
-
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "D3D12_Simple_Triangle", 5)
+    demos_test_name = 'D3D12_Simple_Triangle'
 
     def check_capture(self):
         self.check_final_backbuffer()

util/test/tests/GL/GL_Buffer_Updates.py

@@ -3,8 +3,7 @@ import renderdoc as rd
 
 
 class GL_Buffer_Updates(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "GL_Buffer_Updates", 5)
+    demos_test_name = 'GL_Buffer_Updates'
 
     def check_capture(self):
         # Make an output so we can pick pixels

util/test/tests/GL/GL_CBuffer_Zoo.py

@@ -3,8 +3,7 @@ import renderdoc as rd
 
 
 class GL_CBuffer_Zoo(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "GL_CBuffer_Zoo", 5)
+    demos_test_name = 'GL_CBuffer_Zoo'
 
     def check_capture(self):
         draw = self.find_draw("Draw")

util/test/tests/GL/GL_DX_Interop.py

@@ -2,10 +2,8 @@ import rdtest
 
 
 class GL_DX_Interop(rdtest.TestCase):
-    platform = 'win32'
-
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "DX_Interop", 7)
+    demos_test_name = 'GL_DX_Interop'
+    demos_frame_cap = 7
 
     def check_capture(self):
         self.check_final_backbuffer()

util/test/tests/GL/GL_Entry_Points.py

@@ -3,8 +3,7 @@ import renderdoc as rd
 
 
 class GL_Entry_Points(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "GL_Entry_Points", 5)
+    demos_test_name = 'GL_Entry_Points'
 
     def check_capture(self):
         sdf = self.controller.GetStructuredFile()

util/test/tests/GL/GL_Overlay_Test.py

@@ -3,8 +3,7 @@ import renderdoc as rd
 
 
 class GL_Overlay_Test(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "GL_Overlay_Test", 5)
+    demos_test_name = 'GL_Overlay_Test'
 
     def check_capture(self):
         self.check_final_backbuffer()

util/test/tests/GL/GL_Resource_Lifetimes.py

@@ -3,8 +3,7 @@ import rdtest
 
 
 class GL_Resource_Lifetimes(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "GL_Resource_Lifetimes", 5)
+    demos_test_name = 'GL_Resource_Lifetimes'
 
     def check_capture(self):
         self.check_final_backbuffer()

util/test/tests/GL/GL_Separable_Geometry_Shaders.py

@@ -3,8 +3,7 @@ import rdtest
 
 
 class GL_Separable_Geometry_Shaders(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "GL_Separable_Geometry_Shaders", 5)
+    demos_test_name = 'GL_Separable_Geometry_Shaders'
 
     def check_capture(self):
         draw = self.find_draw("Draw")

util/test/tests/GL/GL_Simple_Triangle.py

@@ -3,8 +3,7 @@ import rdtest
 
 
 class GL_Simple_Triangle(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "GL_Simple_Triangle", 5)
+    demos_test_name = 'GL_Simple_Triangle'
 
     def check_capture(self):
         self.check_final_backbuffer()

util/test/tests/GL/GL_VAO_0.py

@@ -3,8 +3,7 @@ import renderdoc as rd
 
 
 class GL_VAO_0(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "VAO_0", 5)
+    demos_test_name = 'GL_VAO_0'
 
     def check_capture(self):
         self.check_final_backbuffer()

util/test/tests/Vulkan/VK_Adv_CBuffer_Zoo.py

@@ -3,8 +3,7 @@ import renderdoc as rd
 
 
 class VK_Adv_CBuffer_Zoo(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "VK_Adv_CBuffer_Zoo", 5)
+    demos_test_name = 'VK_Adv_CBuffer_Zoo'
 
     def check_capture(self):
         draw = self.find_draw("Draw")

util/test/tests/Vulkan/VK_CBuffer_Zoo.py

@@ -3,8 +3,7 @@ import renderdoc as rd
 
 
 class VK_CBuffer_Zoo(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "VK_CBuffer_Zoo", 5)
+    demos_test_name = 'VK_CBuffer_Zoo'
 
     def check_capture(self):
         draw = self.find_draw("Draw")

util/test/tests/Vulkan/VK_Descriptor_Indexing.py

@@ -3,8 +3,7 @@ import renderdoc as rd
 
 
 class VK_Descriptor_Indexing(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "VK_Descriptor_Indexing", 5)
+    demos_test_name = 'VK_Descriptor_Indexing'
 
     def check_capture(self):
 

util/test/tests/Vulkan/VK_Indirect.py

@@ -4,8 +4,7 @@ import renderdoc as rd
 
 
 class VK_Indirect(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "VK_Indirect", 5)
+    demos_test_name = 'VK_Indirect'
 
     def check_overlay(self, eventId: int, out: rd.ReplayOutput, tex: rd.TextureDisplay, save_data: rd.TextureSave):
         pipe: rd.PipeState = self.controller.GetPipelineState()

util/test/tests/Vulkan/VK_Overlay_Test.py

@@ -3,8 +3,7 @@ import renderdoc as rd
 
 
 class VK_Overlay_Test(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "VK_Overlay_Test", 5)
+    demos_test_name = 'VK_Overlay_Test'
 
     def check_capture(self):
         self.check_final_backbuffer()

util/test/tests/Vulkan/VK_Resource_Lifetimes.py

@@ -3,8 +3,7 @@ import rdtest
 
 
 class VK_Resource_Lifetimes(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "VK_Resource_Lifetimes", 5)
+    demos_test_name = 'VK_Resource_Lifetimes'
 
     def check_capture(self):
         self.check_final_backbuffer()

util/test/tests/Vulkan/VK_Sample_Locations.py

@@ -3,8 +3,7 @@ import rdtest
 
 
 class VK_Sample_Locations(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "VK_Sample_Locations", 5)
+    demos_test_name = 'VK_Sample_Locations'
 
     def check_capture(self):
         draw: rd.DrawcallDescription = self.find_draw("Degenerate")

util/test/tests/Vulkan/VK_Secondary_CmdBuf.py

@@ -3,8 +3,7 @@ import renderdoc as rd
 
 
 class VK_Secondary_CmdBuf(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "Secondary_CmdBuf", 5)
+    demos_test_name = 'VK_Secondary_CmdBuf'
 
     def check_capture(self):
         self.check_final_backbuffer()

util/test/tests/Vulkan/VK_Simple_Triangle.py

@@ -3,8 +3,7 @@ import rdtest
 
 
 class VK_Simple_Triangle(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "VK_Simple_Triangle", 5)
+    demos_test_name = 'VK_Simple_Triangle'
 
     def check_capture(self):
         self.check_final_backbuffer()

util/test/tests/Vulkan/VK_VS_Max_Desc_Set.py

@@ -3,8 +3,7 @@ import renderdoc as rd
 
 
 class VK_VS_Max_Desc_Set(rdtest.TestCase):
-    def get_capture(self):
-        return rdtest.run_and_capture("demos_x64", "VS_Max_Desc_Set", 5)
+    demos_test_name = 'VK_VS_Max_Desc_Set'
 
     def check_capture(self):