lee/renderdoc
1
0
Fork 0
mirror of https://github.com/baldurk/renderdoc.git synced 2026-09-14 01:35:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ability to run tests in parallel

Browse Source
This commit is contained in:
baldurk
2026-09-11 19:19:26 +01:00
parent 73545f0a51
commit fc27cf0deb
24 changed files with 188 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
renderdoc/replay/entry_points.cpp
-2
View File
@@ -1012,8 +1012,6 @@ extern "C" RENDERDOC_API int RENDERDOC_CC RENDERDOC_RunFunctionalTests(const rdc
// specify python module path
L"--pyrenderdoc",
StringFormat::UTF82Wide(modulePath),
// force in-process as we can't fork out to python to pass args
L"--in-process",
});
rdcarray<wchar_t *> wideArgStrings;
util/test/rdtest/logging.py
+26 -8
View File
@@ -4,6 +4,7 @@ import sys
import re
import traceback
import mimetypes
import threading
import difflib
import shutil
from typing import Any, List, Type
@@ -32,11 +33,22 @@ class TestLogger:
self.failed = False
self.section_failed = False
self.logged_exception = False
self.mutex = threading.Lock()
def subprocess_print(self, line: str):
for o in self.outputs:
o.write(line)
o.flush()
def subprocess_test(self, test: str, thread: int, out_buf: List[str], path: str, exc: Exception | None = None):
with self.mutex:
self.begin_test(test, True, thread)
with open(path) as f:
lines = f.readlines()
for l in lines:
self.rawprint(l, with_stdout=False)
sys.stdout.write(out_buf[0])
sys.stdout.flush()
sys.stderr.write(out_buf[1])
sys.stderr.flush()
if exc is not None:
self.failure(exc)
self.end_test(test, True, thread)
def rawprint(self, line: str, with_stdout=True):
for o in self.outputs:
@@ -70,21 +82,27 @@ class TestLogger:
def dedent(self):
self.indentation -= 4
def begin_test(self, test_name: str, print_header: bool=True):
def begin_test(self, test_name: str, print_header: bool=True, thread=-1):
self.test_name = test_name
if print_header:
self.rawprint(f">> Test {test_name}")
if thread >= 0:
self.rawprint(f">> Test {test_name} (Worker {thread})")
else:
self.rawprint(f">> Test {test_name}")
self.indent()
self.failed = False
self.logged_exception = False
def end_test(self, test_name: str, print_footer: bool=True):
def end_test(self, test_name: str, print_footer: bool=True, thread=-1):
if self.failed:
self.rawprint("$$ FAILED")
self.dedent()
if print_footer:
self.rawprint(f"<< Test {test_name}")
if thread >= 0:
self.rawprint(f"<< Test {test_name} (Worker {thread})")
else:
self.rawprint(f"<< Test {test_name}")
self.test_name = ''
def begin_section(self, name: str):
util/test/rdtest/remoteserver.py
+3 -3
View File
@@ -19,7 +19,7 @@ class RemoteServer(ABC):
self.remote: rd.RemoteServer | None = None
@abstractmethod
def init(self, in_process: bool):
def init(self, debugger: bool):
pass
@abstractmethod
@@ -115,7 +115,7 @@ class AndroidRemoteServer(RemoteServer):
self.remote = None
self._base_path = ''
def init(self, in_process):
def init(self, debugger):
# Remove any existing Vulkan layers
subprocess.run(['adb', '-s', self.device,
'shell', 'settings', 'delete', 'global', 'gpu_debug_layers'], check=False)
@@ -134,7 +134,7 @@ class AndroidRemoteServer(RemoteServer):
# Close the connection if the tests are forked as each test will create their own
# connection
if not in_process:
if not debugger:
self.disconnect()
def connect(self):
util/test/rdtest/runner.py
+128 -54
View File
@@ -48,7 +48,10 @@ def _enqueue_output(process: subprocess.Popen[str], out: IO[str], q: queue.Queue
pass
def _run_test(testclass: TestCaseType, runner_timeout: int, failedcases: List[TestCaseType]):
KEYBOARD_EXIT = 100
def _run_test(testclass: TestCaseType, thread: int, runner_timeout: int, out_buf: List[str] | None, failedcases: List[TestCaseType]):
name = testclass.__name__
# Fork the interpreter to run the test, in case it crashes we can catch it.
@@ -57,6 +60,8 @@ def _run_test(testclass: TestCaseType, runner_timeout: int, failedcases: List[Te
args.insert(0, sys.executable)
# Add parameter to run the test itself
args.append('--internal_thread')
args.append(str(thread))
args.append('--internal_run_test')
args.append(name)
@@ -83,6 +88,7 @@ def _run_test(testclass: TestCaseType, runner_timeout: int, failedcases: List[Te
out_pending = ""
err_pending = ""
timeout = False
while test_run.poll() is None:
out = err = ""
@@ -124,33 +130,41 @@ def _run_test(testclass: TestCaseType, runner_timeout: int, failedcases: List[Te
if err is not None:
err_pending += err
while True:
try:
nl = out_pending.index('\n')
line = out_pending[0:nl]
out_pending = out_pending[nl+1:]
line = line.replace('\r', '')
sys.stdout.write(line + '\n')
sys.stdout.flush()
except:
break
if out_buf is None:
while True:
try:
nl = out_pending.index('\n')
line = out_pending[0:nl]
out_pending = out_pending[nl+1:]
line = line.replace('\r', '')
sys.stdout.write(line + '\n')
sys.stdout.flush()
except:
break
while True:
try:
nl = err_pending.index('\n')
line = err_pending[0:nl]
err_pending = err_pending[nl+1:]
line = line.replace('\r', '')
sys.stderr.write(line + '\n')
sys.stderr.flush()
except:
break
while True:
try:
nl = err_pending.index('\n')
line = err_pending[0:nl]
err_pending = err_pending[nl+1:]
line = line.replace('\r', '')
sys.stderr.write(line + '\n')
sys.stderr.flush()
except:
break
if out is None and err is None and test_run.poll() is None:
log.error(f'Timed out, no output within {runner_timeout}s elapsed')
test_run.kill()
test_run.communicate()
raise subprocess.TimeoutExpired(' '.join(args), runner_timeout)
timeout = True
break
if out_buf is not None:
out_buf += [out_pending, err_pending]
if timeout:
raise subprocess.TimeoutExpired(' '.join(args), runner_timeout)
if RUNNER_DEBUG:
print("Test runner has finished")
@@ -175,6 +189,9 @@ def _run_test(testclass: TestCaseType, runner_timeout: int, failedcases: List[Te
# so we just need to mark this test as failed
elif test_run.returncode == 1:
failedcases.append(testclass)
elif test_run.returncode == KEYBOARD_EXIT:
log.print("Propagating keyboard interrupt up from worker")
os._exit(KEYBOARD_EXIT)
else:
raise RuntimeError(f'Test did not exit cleanly while running, possible crash. Exit code {test_run.returncode}')
@@ -191,14 +208,14 @@ def fetch_tests():
return { x[0]: (x[1] == 'True', x[2]) for x in split_tests }
def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests: bool, debugger: bool, test_timeout: int):
def run_tests(test_include: str, test_exclude: str, debugger: bool, parallel: int, test_timeout: int):
start_time = datetime.datetime.now(datetime.timezone.utc)
rd.InitialiseReplay(rd.GlobalEnvironment(), [])
server = util.get_remote_server()
if server is not None:
server.init(in_process)
server.init(debugger)
# On windows, disable error reporting
if 'windll' in dir(ctypes):
@@ -228,6 +245,20 @@ def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests
log.header(f"Tests running for RenderDoc Version {rd.GetVersionString()} ({rd.GetCommitHash()})")
log.header(f"On {platform.platform()}")
# make parallel=0 for no parallelism so we can use it as bool flag
if parallel <= 1:
parallel = 0
if debugger:
if parallel:
log.print(f"Disabling parallel={parallel} for debugging")
parallel = 0
if parallel:
log.header(f"With {parallel} parallel test runners")
else:
log.header(f"With serial test runners")
log.comment(f"plat={platform.platform()} git={rd.GetCommitHash()}")
log.print(f"Demos running from {util.get_demos_binary()}")
@@ -336,36 +367,66 @@ def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests
skippedcases.append(testclass)
continue
if not slow_tests and testclass.slow_test:
log.print(f"Skipping {name} as it is a slow test, which are not enabled")
skippedcases.append(testclass)
continue
runcases.append((testclass, name, instance))
for testclass, name, instance in runcases:
# Print header (and footer) outside the exec so we know they will always be printed successfully
log.begin_test(name)
util.set_current_test(name)
def do(debugMode: bool):
if in_process:
instance.invoketest(debugMode)
else:
_run_test(testclass, test_timeout, failedcases)
if debugger:
do(True)
def test_runner(thread: int):
if parallel:
tests_to_run = [runcases[i] for i in range(thread, len(runcases), parallel)]
else:
try:
do(False)
except Exception as ex:
log.failure(ex)
failedcases.append(testclass)
tests_to_run = runcases
log.end_test(name)
for testclass, name, instance in tests_to_run:
output_buf: List[str] | None = []
# Print header (and footer) outside the exec so we know they will always be printed successfully
if not parallel:
log.begin_test(name)
output_buf = None
def do():
nonlocal output_buf
# don't exec if we're not running from python
if debugger or "python" not in os.path.basename(sys.executable):
util.set_current_test(name)
instance.invoketest(debugger)
else:
_run_test(testclass, thread, test_timeout, output_buf, failedcases)
if debugger:
do()
else:
try:
do()
if parallel:
assert output_buf is not None
log.subprocess_test(name, thread, output_buf, util.get_tmp_path("output.log.html", name))
except KeyboardInterrupt as ex:
log.print("Detected keyboard interrupt in harness - exiting")
os._exit(KEYBOARD_EXIT)
except Exception as ex:
if parallel:
assert output_buf is not None
log.subprocess_test(name, thread, output_buf, util.get_tmp_path("output.log.html", name), ex)
else:
log.failure(ex)
failedcases.append(testclass)
if not parallel:
log.end_test(name)
if not parallel:
test_runner(-1)
else:
threads = [threading.Thread(target=test_runner, args=(k,)) for k in range(parallel)]
[t.start() for t in threads]
while any([t.is_alive() for t in threads]):
[t.join(5) for t in threads if t.is_alive()]
duration = datetime.datetime.now(datetime.timezone.utc) - start_time
if server is not None:
@@ -409,6 +470,8 @@ def vulkan_register():
rd.UpdateVulkanLayerRegistration(True)
FIRST_REMOTE_SERVER_PORT = 39930
def launch_remote_server():
# Fork the interpreter to run the test, in case it crashes we can catch it.
# We can re-run with the same parameters
@@ -430,14 +493,16 @@ def launch_remote_server():
args.insert(2, 'functional')
subprocess.Popen(args)
return
return FIRST_REMOTE_SERVER_PORT
def become_remote_server():
rd.BecomeRemoteServer('localhost', 0, None, None)
def become_remote_server(thread: int):
if thread == -1:
thread = 0
rd.BecomeRemoteServer('localhost', FIRST_REMOTE_SERVER_PORT+thread, None, None)
def internal_run_test(test_name: str):
def internal_run_test(thread: int, test_name: str):
# In case of out-of-process testing, connect to the server
server = util.get_remote_server()
if server is not None:
@@ -445,7 +510,12 @@ def internal_run_test(test_name: str):
testcases = get_tests()
log.add_output(util.get_artifact_path("output.log.html"))
# if we're not running in parallel write directly to the output log
if thread == -1:
log.add_output(util.get_artifact_path("output.log.html"))
thread = 0
else:
log.add_output(util.get_tmp_path("output.log.html", test_name))
for testclass in testcases:
if testclass.__name__ == test_name:
@@ -459,8 +529,12 @@ def internal_run_test(test_name: str):
try:
instance = testclass()
instance.worker_thread = thread
instance.invoketest(False)
suceeded = True
except KeyboardInterrupt:
log.print("Detected keyboard interrupt in test worker - exiting")
os._exit(KEYBOARD_EXIT)
except Exception as ex:
log.failure(ex)
suceeded = False
util/test/rdtest/shared/Groupshared.py
+1 -1
View File
@@ -4,6 +4,7 @@ import rdtest
class Groupshared(rdtest.TestCase):
internal = True
slow_test = True
demos_test_name = None
def check_compute_thread_result(self, test: int, action: rd.ActionDescription, x: int, y: int, z: int, expected: rdtest.VectorValue):
@@ -108,6 +109,5 @@ class Groupshared(rdtest.TestCase):
action = self.find_action("Compute Tests")
assert action is not None
self.check_compute_section_tests(action)
self.check_renderdoc_log_asserts()
rdtest.log.success("All tests matched")
util/test/rdtest/shared/Pixel_History.py
+1
View File
@@ -16,6 +16,7 @@ PropGetter = Callable[[rd.PixelModification], Any]
class Pixel_History(rdtest.TestCase):
internal = True
slow_test = True
demos_test_name = None
def check_capture(self):
util/test/rdtest/shared/Subgroup_Zoo.py
+1 -2
View File
@@ -7,6 +7,7 @@ import rdtest
# Not a real test, re-used by API-specific tests
class Subgroup_Zoo(rdtest.TestCase):
internal = True
slow_test = True
demos_test_name = None
workgroup = (0, 0, 0)
@@ -275,6 +276,4 @@ class Subgroup_Zoo(rdtest.TestCase):
if overallFailed:
raise rdtest.TestFailureException("Some tests were not as expected")
self.check_renderdoc_log_asserts()
rdtest.log.success("All tests matched")
util/test/rdtest/shared/Texture_Zoo.py
+5 -4
View File
@@ -23,6 +23,7 @@ def linear2srgb(f: float):
# Not a real test, re-used by API-specific tests
class Texture_Zoo():
def __init__(self):
self.worker_thread = 0
self.proxied = False
self.fake_msaa = False
self.filename = ''
@@ -675,20 +676,20 @@ class Texture_Zoo():
self.controller = None
# Launch a remote server
rdtest.launch_remote_server()
base_port = rdtest.launch_remote_server()
# Wait for it to start
time.sleep(0.5)
result, remote = rd.CreateRemoteServerConnection('localhost')
result, remote = rd.CreateRemoteServerConnection(f'localhost:{base_port+self.worker_thread}')
if not result:
time.sleep(2)
result, remote = rd.CreateRemoteServerConnection('localhost')
result, remote = rd.CreateRemoteServerConnection(f'localhost:{base_port+self.worker_thread}')
if not result:
raise rdtest.TestFailureException(f"Couldn't connect to remote server: {result!s}")
raise rdtest.TestFailureException(f"Couldn't connect to remote server {base_port+self.worker_thread}: {result!s}")
proxies = remote.LocalProxies()
util/test/rdtest/shared/Workgroup_Zoo.py
+1 -2
View File
@@ -3,6 +3,7 @@ import rdtest
# Not a real test, re-used by API-specific tests
class Workgroup_Zoo(rdtest.Subgroup_Zoo):
internal = True
slow_test = True
demos_test_name = None
def check_capture(self):
@@ -28,6 +29,4 @@ class Workgroup_Zoo(rdtest.Subgroup_Zoo):
if self.check_compute_tests(compute_dims, thread_checks):
raise rdtest.TestFailureException("Some tests were not as expected")
self.check_renderdoc_log_asserts()
rdtest.log.success("All tests matched")
util/test/rdtest/testcase.py
+1 -11
View File
@@ -163,6 +163,7 @@ class TestCase:
def __init__(self):
self.capture_filename = ""
self.worker_thread = 0
self.controller: rd.ReplayController | None = None
self.sdfile: rd.SDFile | None = None
self._variables = []
@@ -1132,17 +1133,6 @@ class TestCase:
taskIdx += 1
return data
def check_renderdoc_log_asserts(self):
countAsserts = 0
rdlog = rd.GetLogFile()
with open(rdlog, 'r') as f:
for line in f:
if 'Assertion' in line:
log.error(line)
countAsserts += 1
if countAsserts > 0:
raise TestFailureException(f'Renderdoc log file contains {countAsserts} Asserts')
def validate_shadervariable(self, var: rd.ShaderVariable):
if len(var.members) != 0:
if var.type != rd.VarType.Struct and var.type != rd.VarType.Unknown and var.type != rd.VarType.ConstantBlock:
util/test/rdtest/util.py
+5 -3
View File
@@ -158,9 +158,11 @@ def get_demos_timeout():
return _demos_timeout
def get_tmp_path(name: str):
os.makedirs(os.path.join(_temp_dir, _test_name), exist_ok=True)
return os.path.join(_temp_dir, _test_name, name)
def get_tmp_path(name: str, test = ""):
if test == "":
test = get_current_test()
os.makedirs(os.path.join(_temp_dir, test), exist_ok=True)
return os.path.join(_temp_dir, test, name)
def get_android_demo_app_name():
util/test/run_tests.py
+6 -10
View File
@@ -15,10 +15,8 @@ parser.add_argument('-t', '--test_include', default=".*",
help="The tests to include, as a regexp filter", type=str)
parser.add_argument('-x', '--test_exclude', default="",
help="The tests to exclude, as a regexp filter", type=str)
parser.add_argument('--in-process',
help="Run test code in the same process as test runner", action="store_true")
parser.add_argument('--slow-tests',
help="Run potentially slow tests", action="store_true")
parser.add_argument('-j', '--parallel',
help="Run test in N processes in parallel where possible", default=0, type=int)
parser.add_argument('--test-timeout',
help="Timeout for output from tests", default=90, type=int)
parser.add_argument('--data', default=os.path.join(script_dir, "data"),
@@ -44,6 +42,7 @@ parser.add_argument('--internal_run_test', help=argparse.SUPPRESS, type=str, req
parser.add_argument('--internal_vulkan_register', help=argparse.SUPPRESS, action="store_true", required=False)
# Internal command, when we re-run as a remote server
parser.add_argument('--internal_remote_server', help=argparse.SUPPRESS, action="store_true", required=False)
parser.add_argument('--internal_thread', help=argparse.SUPPRESS, type=int, default=0, required=False)
args = parser.parse_args()
custom_pyrenderdoc = None
@@ -158,15 +157,12 @@ if args.adb_device:
rdtest.create_adb_device(args.adb_device)
else:
rdtest.set_remote_server(None)
# debugger option implies in-process test running
if args.debugger:
args.in_process = True
if args.internal_vulkan_register:
rdtest.vulkan_register()
elif args.internal_remote_server:
rdtest.become_remote_server()
rdtest.become_remote_server(args.internal_thread)
elif args.internal_run_test is not None:
rdtest.internal_run_test(args.internal_run_test)
rdtest.internal_run_test(args.internal_thread, args.internal_run_test)
else:
rdtest.run_tests(args.test_include, args.test_exclude, args.in_process, args.slow_tests, args.debugger, args.test_timeout)
rdtest.run_tests(args.test_include, args.test_exclude, args.debugger, args.parallel, args.test_timeout)
util/test/tests/D3D11/D3D11_Shader_Debug_Zoo.py
+1
View File
@@ -4,6 +4,7 @@ import rdtest
class D3D11_Shader_Debug_Zoo(rdtest.TestCase):
demos_test_name = 'D3D11_Shader_Debug_Zoo'
slow_test = True
def check_capture(self):
undefined_tests = [int(test) for test in self.find_action("Undefined tests: ").customName.split(" ")[2:]]
util/test/tests/D3D11/D3D11_Texture_Zoo.py
+1
View File
@@ -12,5 +12,6 @@ class D3D11_Texture_Zoo(rdtest.TestCase):
def check_capture(self):
assert self.controller is not None
# This takes ownership of the controller and shuts it down when it's finished
self.zoo_helper.worker_thread = self.worker_thread
self.zoo_helper.check_capture(self.capture_filename, self.controller)
self.controller = None
util/test/tests/D3D12/D3D12_Execute_Indirect.py
+1
View File
@@ -5,6 +5,7 @@ from typing import List
class D3D12_Execute_Indirect(rdtest.TestCase):
demos_test_name = 'D3D12_Execute_Indirect'
slow_test = True
def check_overlays(self, eid: int, x: int, y: int):
with rdtest.log.auto_section(f'EID {eid} Checking Overlays at {x}, {y}'):
util/test/tests/D3D12/D3D12_Groupshared.py
-2
View File
@@ -16,6 +16,4 @@ class D3D12_Groupshared(rdtest.Groupshared):
if overallFailed:
raise rdtest.TestFailureException("Some tests were not as expected")
self.check_renderdoc_log_asserts()
rdtest.log.success("All tests matched")
util/test/tests/D3D12/D3D12_Shader_DebugData_Zoo.py
-2
View File
@@ -311,6 +311,4 @@ class D3D12_Shader_DebugData_Zoo(rdtest.TestCase):
if failed:
raise rdtest.TestFailureException("Some tests were not as expected")
self.check_renderdoc_log_asserts()
rdtest.log.success("All tests matched")
util/test/tests/D3D12/D3D12_Shader_Debug_Zoo.py
+1
View File
@@ -4,6 +4,7 @@ import struct
class D3D12_Shader_Debug_Zoo(rdtest.TestCase):
demos_test_name = 'D3D12_Shader_Debug_Zoo'
slow_test = True
def check_compute_derivative_tests(self):
failed = False
util/test/tests/D3D12/D3D12_Texture_Zoo.py
+1
View File
@@ -12,5 +12,6 @@ class D3D12_Texture_Zoo(rdtest.TestCase):
def check_capture(self):
assert self.controller is not None
# This takes ownership of the controller and shuts it down when it's finished
self.zoo_helper.worker_thread = self.worker_thread
self.zoo_helper.check_capture(self.capture_filename, self.controller)
self.controller = None
util/test/tests/GL/GL_Callstacks.py
+1
View File
@@ -13,6 +13,7 @@ def resolve_progress(progress: float):
class GL_Callstacks(rdtest.TestCase):
demos_test_name = 'GL_Callstacks'
slow_test = True
def get_capture_options(self):
ret = rd.CaptureOptions()
util/test/tests/GL/GL_Shader_Debug_Zoo.py
+1
View File
@@ -4,6 +4,7 @@ import rdtest
class GL_Shader_Debug_Zoo(rdtest.TestCase):
demos_test_name = 'GL_Shader_Debug_Zoo'
slow_test = True
def check_capture(self):
assert self.controller is not None
util/test/tests/GL/GL_Texture_Zoo.py
+1
View File
@@ -12,5 +12,6 @@ class GL_Texture_Zoo(rdtest.TestCase):
def check_capture(self):
assert self.controller is not None
# This takes ownership of the controller and shuts it down when it's finished
self.zoo_helper.worker_thread = self.worker_thread
self.zoo_helper.check_capture(self.capture_filename, self.controller)
self.controller = None
util/test/tests/Vulkan/VK_Shader_Debug_Zoo.py
+1
View File
@@ -6,6 +6,7 @@ import struct
class VK_Shader_Debug_Zoo(rdtest.TestCase):
demos_test_name = 'VK_Shader_Debug_Zoo'
slow_test = True
def check_capture(self):
if not self.controller.GetAPIProperties().shaderDebugging:
util/test/tests/Vulkan/VK_Texture_Zoo.py
+1
View File
@@ -12,5 +12,6 @@ class VK_Texture_Zoo(rdtest.TestCase):
def check_capture(self):
assert self.controller is not None
# This takes ownership of the controller and shuts it down when it's finished
self.zoo_helper.worker_thread = self.worker_thread
self.zoo_helper.check_capture(self.capture_filename, self.controller)
self.controller = None