From b1d3123583daf0ebe5b47b3318c9aaa2bdb98c24 Mon Sep 17 00:00:00 2001 From: thisisjimmyfb <58957694+thisisjimmyfb@users.noreply.github.com> Date: Tue, 3 Jun 2025 00:52:36 -0700 Subject: [PATCH] Remote Android and Linux test support part 1 Add remote server support to test framework --- util/test/README.md | 1 + util/test/demos/main.cpp | 26 +-- util/test/demos/test_common.cpp | 27 +++ util/test/demos/test_common.h | 1 + util/test/rdtest/analyse.py | 43 ++-- util/test/rdtest/capture.py | 35 +++- util/test/rdtest/remoteserver.py | 335 +++++++++++++++++++++++++++++++ util/test/rdtest/runner.py | 154 ++++++++------ util/test/rdtest/testcase.py | 31 ++- util/test/rdtest/util.py | 52 +++++ util/test/run_tests.py | 6 + 11 files changed, 612 insertions(+), 99 deletions(-) create mode 100644 util/test/rdtest/remoteserver.py diff --git a/util/test/README.md b/util/test/README.md index 9c412046d..0366c9821 100644 --- a/util/test/README.md +++ b/util/test/README.md @@ -42,6 +42,7 @@ Then running the tests means invoking `run_tests.py` with any options you need: * `--temp` the path to the temporary working folder, by default `tmp/` here next to the script. * `--data-extra` the path to the extra data folder. Some tests may reference captures which can't be committed to the repository here and are distributed separately or added custom by the user. By default refers to `data_extra/` here next to the script. * `--demos-binary` the path to the built demos binary. +* `--adb-device` the ADB device to run the tests on, instead of the host. If set, --demos-binary is expected to be the demo APK **NOTE:** When run, the temporary and artifacts folders will be erased. diff --git a/util/test/demos/main.cpp b/util/test/demos/main.cpp index 5622ee1be..70fa5cfd2 100644 --- a/util/test/demos/main.cpp +++ b/util/test/demos/main.cpp @@ -395,7 +395,7 @@ int main(int argc, char **argv) if(argc >= 2 && (!strcmp(argv[1], "--help") || !strcmp(argv[1], "-h") || !strcmp(argv[1], "-?") || !strcmp(argv[1], "/help") || !strcmp(argv[1], "/h") || !strcmp(argv[1], "/?"))) { - printf(R"(RenderDoc testing demo program + OutputPrint(R"(RenderDoc testing demo program Usage: %s Test_Name [test_options] @@ -417,9 +417,8 @@ Usage: %s Test_Name [test_options] environment variable, or else in the data/demos folder next to the executable. )", - argc == 0 ? "demos" : argv[0]); + argc == 0 ? "demos" : argv[0]); - fflush(stdout); return 1; } @@ -434,21 +433,20 @@ Usage: %s Test_Name [test_options] if(test.API != prev) { if(prev != TestAPI::Count) - printf("\n\n"); - printf("======== %s tests ========\n\n", APIName(test.API)); + OutputPrint("\n\n"); + OutputPrint("======== %s tests ========\n\n", APIName(test.API)); } prev = test.API; - printf("%s: %s", test.Name, test.IsAvailable() ? "Available" : "Unavailable"); + OutputPrint("%s: %s", test.Name, test.IsAvailable() ? "Available" : "Unavailable"); if(!test.IsAvailable()) - printf(" because %s", test.AvailMessage()); + OutputPrint(" because %s", test.AvailMessage()); - printf("\n\t%s\n\n", test.Description); + OutputPrint("\n\t%s\n\n", test.Description); } - fflush(stdout); return 1; } @@ -459,15 +457,14 @@ Usage: %s Test_Name [test_options] check_tests(argc, argv); // output TSV - printf("Name\tAvailable\tAvailMessage\n"); + OutputPrint("Name\tAvailable\tAvailMessage\n"); for(const TestMetadata &test : tests) { - printf("%s\t%s\t%s\n", test.Name, test.IsAvailable() ? "True" : "False", - test.IsAvailable() ? "Available" : test.AvailMessage()); + OutputPrint("%s\t%s\t%s\n", test.Name, test.IsAvailable() ? "True" : "False", + test.IsAvailable() ? "Available" : test.AvailMessage()); } - fflush(stdout); return 1; } @@ -697,7 +694,6 @@ Usage: %s Test_Name [test_options] } TEST_ERROR("%s is not a known test", argv[1]); - return 2; } @@ -745,7 +741,7 @@ int WINAPI wWinMain(_In_ HINSTANCE hInst, _In_opt_ HINSTANCE hPrevInstance, _In_ struct android_app *android_state; pthread_t cmdthread_handle = 0; -#define ANDROID_LOG(...) __android_log_print(ANDROID_LOG_INFO, "rd_demos", __VA_ARGS__); +#define ANDROID_LOG(...) __android_log_print(ANDROID_LOG_DEBUG, "rd_demos", __VA_ARGS__); std::vector getArgs() { diff --git a/util/test/demos/test_common.cpp b/util/test/demos/test_common.cpp index 7454a3c8a..f5d6acd90 100644 --- a/util/test/demos/test_common.cpp +++ b/util/test/demos/test_common.cpp @@ -190,6 +190,33 @@ void DebugPrint(const char *fmt, ...) OutputDebugStringA(printBuf); #endif +#if defined(ANDROID) + __android_log_print(ANDROID_LOG_DEBUG, "rd_demos", "%s", printBuf); +#endif +} + +void OutputPrint(const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + + vsnprintf(printBuf, 4095, fmt, args); + + va_end(args); + + fputs(printBuf, stdout); + fflush(stdout); + + if(logFile) + { + fputs(printBuf, logFile); + fflush(logFile); + } + +#if defined(WIN32) + OutputDebugStringA(printBuf); +#endif + #if defined(ANDROID) __android_log_print(ANDROID_LOG_INFO, "rd_demos", "%s", printBuf); #endif diff --git a/util/test/demos/test_common.h b/util/test/demos/test_common.h index 3ff413945..00b288730 100644 --- a/util/test/demos/test_common.h +++ b/util/test/demos/test_common.h @@ -337,6 +337,7 @@ std::string trim(const std::string &str); void SetDebugLogEnabled(bool enabled); void DebugPrint(const char *fmt, ...); +void OutputPrint(const char *fmt, ...); #define TEST_ASSERT(cond, fmt, ...) \ do \ diff --git a/util/test/rdtest/analyse.py b/util/test/rdtest/analyse.py index 3a085c4b9..0e5e3b1c2 100644 --- a/util/test/rdtest/analyse.py +++ b/util/test/rdtest/analyse.py @@ -1,6 +1,7 @@ import struct from typing import List import renderdoc +from . import util # Alias for convenience - we need to import as-is so types don't get confused rd = renderdoc @@ -24,27 +25,39 @@ def open_capture(filename="", cap: rd.CaptureFile=None, opts: rd.ReplayOptions=N own_cap = False api = "Unknown" - if cap is None: - own_cap = True + result = None + controller = None + if util.get_remote_server() is None: + if cap is None: + own_cap = True - cap = rd.OpenCaptureFile() + cap = rd.OpenCaptureFile() - # Open a particular file - result = cap.OpenFile(filename, '', None) + # Open a particular file + result = cap.OpenFile(filename, '', None) - # Make sure the file opened successfully - if result != rd.ResultCode.Succeeded: - cap.Shutdown() - raise RuntimeError("Couldn't open '{}': {}".format(filename, str(result))) + # Make sure the file opened successfully + if result != rd.ResultCode.Succeeded: + cap.Shutdown() + raise RuntimeError("Couldn't open '{}': {}".format(filename, str(result))) - api = cap.DriverName() + api = cap.DriverName() - # Make sure we can replay - if not cap.LocalReplaySupport(): - cap.Shutdown() - raise RuntimeError("{} capture cannot be replayed".format(api)) + # Make sure we can replay + if not cap.LocalReplaySupport(): + cap.Shutdown() + raise RuntimeError("{} capture cannot be replayed".format(api)) - result, controller = cap.OpenCapture(opts, None) + result, controller = cap.OpenCapture(opts, None) + else: + if not cap is None: + raise ValueError("Cannot call analyse.open_capture() with capture handle for remote {}" + .format(util.get_remote_server().remote)) + + result, controller = util.get_remote_server().remote.OpenCapture(rd.RemoteServer.NoPreference, + filename, opts, None) + if result == rd.ResultCode.Succeeded: + api = util.get_remote_server().remote.DriverName() if own_cap: cap.Shutdown() diff --git a/util/test/rdtest/capture.py b/util/test/rdtest/capture.py index e2562c5cf..b33897fb9 100644 --- a/util/test/rdtest/capture.py +++ b/util/test/rdtest/capture.py @@ -5,6 +5,7 @@ import time import renderdoc as rd from . import util from .logging import log +from time import sleep class TargetControl(): @@ -134,10 +135,14 @@ def run_executable(exe: str, cmdline: str, wait_for_exit = False - log.print("Running exe:'{}' cmd:'{}' in dir:'{}' with env:'{}'".format(exe, cmdline, workdir, envmods)) - # Execute the test program - res = rd.ExecuteAndInject(exe, workdir, cmdline, envmods, cappath, opts, wait_for_exit) + server = util.get_remote_server() + res = None + if server is None: + log.print("Running exe:'{}' cmd:'{}' in dir:'{}' with env:'{}'".format(exe, cmdline, workdir, envmods)) + res = rd.ExecuteAndInject(exe, workdir, cmdline, envmods, cappath, opts, wait_for_exit) + else: + res = server.inject_and_run_exe(cmdline, envmods, opts) if res.result != rd.ResultCode.Succeeded: raise RuntimeError("Couldn't launch program: {}".format(str(res.result))) @@ -171,7 +176,20 @@ def run_and_capture(exe: str, cmdline: str, frame: int, *, frame_count=1, captur if captures_expected is None: captures_expected = frame_count - control = TargetControl(run_executable(exe, cmdline, cappath=util.get_tmp_path(capture_name), opts=opts), timeout=timeout) + host = "localhost" + username = "testrunner" + cappath = "" + + server = util.get_remote_server() + if server is not None: + cappath = server.get_temp_path(capture_name) + host = server.get_hostname() + username = server.get_username() + else: + cappath = util.get_tmp_path(capture_name) + + control = TargetControl(run_executable(exe, cmdline, cappath=cappath, opts=opts), + host=host, username=username, timeout=timeout) log.print("Queuing capture of frame {}..{} with timeout of {}".format(frame, frame+frame_count, "default" if timeout is None else timeout)) @@ -183,6 +201,15 @@ def run_and_capture(exe: str, cmdline: str, frame: int, *, frame_count=1, captur control.run(keep_running=lambda x: len(x.captures()) < captures_expected) captures = control.captures() + log.print(f'Retrieved {len(captures)} captures') + + # Retrieve the demo logfile from the remote device + if server is not None: + remote_logfile = server.get_temp_path('demos.log') + if server.path_exists(remote_logfile): + log.print("Copying remote demo log from '{}' to '{}'".format(server.get_temp_path('demos.log'), logfile)) + os.makedirs(os.path.dirname(logfile), exist_ok=True) + server.remote.CopyCaptureFromRemote(server.get_temp_path('demos.log'), logfile, None) if logfile is not None and os.path.exists(logfile): log.inline_file('Process output', logfile, with_stdout=True) diff --git a/util/test/rdtest/remoteserver.py b/util/test/rdtest/remoteserver.py new file mode 100644 index 000000000..99d70b99b --- /dev/null +++ b/util/test/rdtest/remoteserver.py @@ -0,0 +1,335 @@ + +import sys +import subprocess +import renderdoc as rd +from . import util +from .logging import log +from pathlib import Path +import os +import re + +from time import sleep +from abc import ABC, abstractmethod + + +class RemoteServer(ABC): + def __init__(self) -> None: + super().__init__() + self.device = None + self.remote = None + + @abstractmethod + def init(self, in_process): + pass + + @abstractmethod + def connect(self): + pass + + @abstractmethod + def disconnect(self): + pass + + @abstractmethod + def shutdown(self): + pass + + @abstractmethod + def is_connected(self): + pass + + @abstractmethod + def get_temp_path(self, name, timeout): + pass + + @abstractmethod + def get_renderdoc_path(self): + pass + + @abstractmethod + def path_exists(self, path, timeout): + pass + + @abstractmethod + def run_demos(self, args, timeout): + pass + + @abstractmethod + def inject_and_run_exe(self, cmdline, envmods, opts): + pass + + @abstractmethod + def get_demos_exe(self): + pass + + @abstractmethod + def get_hostname(self): + pass + + @abstractmethod + def get_username(self): + pass + + @abstractmethod + def retrieve_latest_test_log(self, dst, timeout): + pass + + @abstractmethod + def retrieve_latest_server_log(self, dst, timeout): + pass + + @abstractmethod + def retrieve_comms_log(self, timeout): + pass + + +class AndroidRemoteServer(RemoteServer): + # Android app IDs for the server + ADRD_SERVER_APP64 = 'org.renderdoc.renderdoccmd.arm64' + CONNECTION_RETRY_COUNT = 3 + + def __init__(self, device) -> None: + super().__init__() + if device is None: + raise RuntimeError('Android target specified, but no device given') + self.device = device + self.remote = None + self._base_path = '' + + def init(self, in_process): + # Remove any existing Vulkan layers + subprocess.run(['adb', '-s', self.device, + 'shell', 'settings', 'delete', 'global', 'gpu_debug_layers'], check=False) + + remote = self.connect() + log.print(f'Supported drivers: {remote.RemoteSupportedReplays()}') + + # Install the demo APK + subprocess.run(['adb', '-s', self.device, + 'install', '-g', util.get_demos_binary()], check=True) + + # Remove any stale logs and captures from previous runs + rm_glob = self._base_path + '/*' + subprocess.run(['adb', '-s', self.device, 'shell', + 'rm', '-rf', rm_glob], check=True) + + # Close the connection if the tests are forked as each test will create their own + # connection + if not in_process: + self.disconnect() + + def connect(self): + log.print("Connecting to remote server...") + + protocols = rd.GetSupportedDeviceProtocols() + if not 'adb' in protocols: + log.print('ADB requested but not an available device protocol') + sys.exit(1) + + protocol = rd.GetDeviceProtocolController('adb') + if not self.device in protocol.GetDevices(): + log.print(f'ADB device {self.device} requested but not discovered') + sys.exit(1) + + if not protocol.IsSupported(self.device): + log.print(f'ADB device {self.device} is not supported') + sys.exit(1) + + url = f'{protocol.GetProtocolName()}://{self.device}' + result, remote = rd.CreateRemoteServerConnection(url) + if result == rd.ResultCode.NetworkIOFailed and protocol is not None: + log.print("Couldn't connect to remote server, trying to start it") + + result = protocol.StartRemoteServer(url) + if result != rd.ResultCode.Succeeded: + log.print( + f"Couldn't launch remote server, got error {str(result)}") + sys.exit(1) + + # Try to connect again! + result, remote = rd.CreateRemoteServerConnection(url) + + # Retry a few times + if result != rd.ResultCode.Succeeded: + for i in range(1, self.CONNECTION_RETRY_COUNT + 1): + result, remote = rd.CreateRemoteServerConnection(url) + if result == rd.ResultCode.Succeeded: + break + + log.print( + f"Couldn't connect to remote server on attempt #{i}, got error {str(result)}") + if i == self.CONNECTION_RETRY_COUNT: + sys.exit(1) + + # Calculate the remote base path + base = '/sdcard/Android/' + output = subprocess.run(['adb', '-s', self.device, 'shell', 'getprop', 'ro.build.version.sdk'], + stdout=subprocess.PIPE, timeout=10, check=True).stdout + api_version = int(str(output, 'utf-8').strip()) + if api_version >= 30: + base += 'media/' + else: + base += 'data/' + + self._data_path = base + self._base_path = base + util.get_android_demo_app_name() + '/files/' + self.remote = remote + + log.print("Connected!") + return remote + + def disconnect(self): + self.remote.ShutdownConnection() + self.remote = None + + def shutdown(self): + # If we running over ADB, close down the server. This will involve first establishing a + # connection if the tests have been running out-of-process + log.print('Shutting down server...') + + # Kill the demo app first + subprocess.run(['adb', '-s', self.device, 'shell', 'am', 'force-stop', + util.get_android_demo_app_name()]) + + if self.remote is None: + protocol = rd.GetDeviceProtocolController('adb') + url = f'{protocol.GetProtocolName()}://{self.device}' + result, self.remote = rd.CreateRemoteServerConnection(url) + if result != rd.ResultCode.Succeeded: + log.print( + f"Couldn't connect to remote server for shutdown, got error {str(result)}") + return + + self.remote.ShutdownServerAndConnection() + self.remote = None + + def is_connected(self) -> bool: + return self.remote is not None + + def get_temp_path(self, name="", timeout=20): + subprocess.run(['adb', '-s', self.device, 'shell', 'mkdir', '-p', + self._base_path + '/' + util.get_current_test()], + timeout=timeout, + check=True) + return self._base_path + util.get_current_test() + '/' + name + + def get_renderdoc_path(self): + return self._data_path + '/' + AndroidRemoteServer.ADRD_SERVER_APP64 + '/files/RenderDoc/' + + def run_demos(self, args: [str], timeout=10): + raw = subprocess.run(['adb', '-s', self.device, 'shell', 'echo', '$EPOCHREALTIME'], + check=True, stdout=subprocess.PIPE, timeout=timeout).stdout + ts = str(raw, 'utf-8').strip() + # Run the command, blocking + proc = subprocess.run(['adb', '-s', self.device, + 'shell', 'am', 'start', '-W', '-n', f'{util.get_android_demo_app_name()}/.Loader', + '-e', 'demos', 'RenderDoc', '-e', 'rd_demos'] + args, + check=True, stdout=subprocess.DEVNULL) + # Extract the log data + raw = subprocess.run(['adb', '-s', self.device, 'shell', + 'logcat', '-b', 'main', '-v', 'brief', '-T', ts, '-d', 'rd_demos:I', '*:S'], + check=True, stdout=subprocess.PIPE, timeout=timeout).stdout + output = str(raw, 'utf-8') + # Remove the per-line prefix + output = output.splitlines() + result = "" + for line in output: + pos = line.find('): ') + if not line.startswith('I/rd_demos(') or pos == -1: + continue + + result += line[pos+3:] + '\n' + + return result + + def path_exists(self, path, timeout=10): + raw = subprocess.run(['adb', '-s', self.device, 'shell', f'ls {path} >> /dev/null'], + stderr=subprocess.PIPE, timeout=timeout).stderr + raw = str(raw, 'utf-8').strip() + + return len(raw) == 0 + + def inject_and_run_exe(self, cmdline, envmods, opts): + package_and_activity = f"{util.get_android_demo_app_name()}/.Loader" + args = "-e demos RenderDoc -e rd_demos \'\"" + cmdline + "\"\'" + + log.print("Running package:'{}' cmd:'{}' with env:'{}'".format( + package_and_activity, cmdline, envmods)) + res = util.get_remote_server().remote.ExecuteAndInject( + package_and_activity, "", args, envmods, opts) + + if res.result != rd.ResultCode.Succeeded: + raise RuntimeError( + "Couldn't launch program: {}".format(str(res.result))) + + return res + + def get_demos_exe(self): + return util.get_android_demo_app_name() + + def get_hostname(self): + return f"adb://{self.device}" + + def get_username(self): + return "testrunner" + + def retrieve_latest_test_log(self, dst, timeout=10): + if not self.is_connected(): + return None + + src = self._base_path + '/RenderDoc' + raw = subprocess.run(['adb', '-s', self.device, 'shell', f'cd {src}; ls -t RenderDoc_* | head -1'], + check=True, stdout=subprocess.PIPE, timeout=timeout).stdout + latestlog = str(raw, 'utf-8').strip() + if not latestlog: + log.print(f"Cannot find latest remote log from '{src}'") + return None + + os.makedirs(dst, exist_ok=True) + + dst = os.path.join(dst, latestlog) + src = os.path.join(src, latestlog) + log.print(f"Copying remote test log from '{src}' to '{dst}'") + self.remote.CopyCaptureFromRemote(src, dst, None) + + return dst + + def retrieve_latest_server_log(self, dst, timeout=10): + if not self.is_connected(): + return None + + src = self.get_renderdoc_path() + raw = subprocess.run(['adb', '-s', self.device, 'shell', f'cd {src}; ls -t RenderDoc_* | head -1'], + check=True, stdout=subprocess.PIPE, timeout=timeout).stdout + latestlog = str(raw, 'utf-8').strip() + if not latestlog: + log.print(f"Cannot find latest remote log from '{src}'") + return None + + os.makedirs(dst, exist_ok=True) + + dst = os.path.join(dst, latestlog) + src = os.path.join(src, latestlog) + log.print(f"Copying remote server log from '{src}' to '{dst}'") + self.remote.CopyCaptureFromRemote(src, dst, None) + + return dst + + def retrieve_comms_log(self, timeout=10): + if not self.is_connected(): + return None + + src = self.get_renderdoc_path() + "RemoteServer_Server.log" + if not self.path_exists(src): + log.print(f"Cannot find server comms log '{src}'") + return None + + os.makedirs(util.get_tmp_dir(), exist_ok=True) + + dst = os.path.join(util.get_tmp_dir(), 'RenderDoc_Server.log') + log.print("Copying remote server comms log from '{}' to '{}'".format(src, dst)) + self.remote.CopyCaptureFromRemote(src, dst, None) + + return dst + + diff --git a/util/test/rdtest/runner.py b/util/test/rdtest/runner.py index 43f83426e..6fa4a508e 100644 --- a/util/test/rdtest/runner.py +++ b/util/test/rdtest/runner.py @@ -14,6 +14,7 @@ from . import util from . import testcase from .logging import log from pathlib import Path +from rdtest.remoteserver import RemoteServer def get_tests(): @@ -176,11 +177,11 @@ def _run_test(testclass, runner_timeout, failedcases: list): .format(test_run.returncode)) -def fetch_tests(): - output = subprocess.run([util.get_demos_binary(), '--list-raw'], stdout=subprocess.PIPE).stdout - - # Skip the header, grab all the remaining lines - tests = str(output, 'utf-8').splitlines()[1:] +def fetch_tests(): + output = util.run_demo_blocking(['--list-raw']).splitlines() + + # Skip to just past the header, grab all the remaining lines + tests = output[output.index("Name\tAvailable\tAvailMessage")+1:] # Split the TSV values and store split_tests = [ test.split('\t') for test in tests ] @@ -193,6 +194,10 @@ def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests rd.InitialiseReplay(rd.GlobalEnvironment(), []) + server: RemoteServer = util.get_remote_server() + if server is not None: + server.init(in_process) + # On windows, disable error reporting if 'windll' in dir(ctypes): ctypes.windll.kernel32.SetErrorMode(1 | 2) # SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX @@ -222,64 +227,63 @@ def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests log.header("On {}".format(platform.platform())) log.comment("plat={} git={}".format(platform.platform(), rd.GetCommitHash())) - - driver = "" - - for api in rd.GraphicsAPI: - v = rd.GetDriverInformation(api) - log.print("{} driver: {} {}".format(str(api), str(v.vendor), v.version)) - - # Take the first version number we get, but prefer GL as it's universally available and - # Produces a nice version number & device combination - if (api == rd.GraphicsAPI.OpenGL or driver == "") and v.vendor != rd.GPUVendor.Unknown: - driver = v.version - - log.comment("driver={}".format(driver)) - log.print("Demos running from {}".format(util.get_demos_binary())) - layerInfo = rd.VulkanLayerRegistrationInfo() - if rd.NeedVulkanLayerRegistration(layerInfo): - log.print("Vulkan layer needs to be registered: {}".format(str(layerInfo.flags))) - log.print("My JSONs: {}, Other JSONs: {}".format(layerInfo.myJSONs, layerInfo.otherJSONs)) + if server is None: + driver = "" + for api in rd.GraphicsAPI: + v = rd.GetDriverInformation(api) + log.print("{} driver: {} {}".format(str(api), str(v.vendor), v.version)) - # Update the layer registration without doing anything special first - if running automated we might have - # granted user-writable permissions to the system files needed to update. If possible we register at user - # level. - if layerInfo.flags & rd.VulkanLayerFlags.NeedElevation: - rd.UpdateVulkanLayerRegistration(True) - else: - rd.UpdateVulkanLayerRegistration(False) + # Take the first version number we get, but prefer GL as it's universally available and + # Produces a nice version number & device combination + if (api == rd.GraphicsAPI.OpenGL or driver == "") and v.vendor != rd.GPUVendor.Unknown: + driver = v.version - # Check if it succeeded - reg_needed = rd.NeedVulkanLayerRegistration(layerInfo) + log.comment("driver={}".format(driver)) - if reg_needed: - if plat == 'win32': - # On windows, try to elevate. This will mean a UAC prompt - args = sys.argv.copy() - args.append("--internal_vulkan_register") + layerInfo = rd.VulkanLayerRegistrationInfo() + if rd.NeedVulkanLayerRegistration(layerInfo): + log.print("Vulkan layer needs to be registered: {}".format(str(layerInfo.flags))) + log.print("My JSONs: {}, Other JSONs: {}".format(layerInfo.myJSONs, layerInfo.otherJSONs)) - for i in range(len(args)): - if os.path.exists(args[i]): - args[i] = str(Path(args[i]).resolve()) - - if 'renderdoccmd' in sys.executable: - args = ['vulkanlayer', '--register', '--system'] - - ctypes.windll.shell32.ShellExecuteW(None, "runas", sys.executable, ' '.join(args), None, 1) - - time.sleep(10) + # Update the layer registration without doing anything special first - if running automated we might have + # granted user-writable permissions to the system files needed to update. If possible we register at user + # level. + if layerInfo.flags & rd.VulkanLayerFlags.NeedElevation: + rd.UpdateVulkanLayerRegistration(True) else: + rd.UpdateVulkanLayerRegistration(False) + + # Check if it succeeded + reg_needed = rd.NeedVulkanLayerRegistration(layerInfo) + + if reg_needed: + if plat == 'win32': + # On windows, try to elevate. This will mean a UAC prompt + args = sys.argv.copy() + args.append("--internal_vulkan_register") + + for i in range(len(args)): + if os.path.exists(args[i]): + args[i] = str(Path(args[i]).resolve()) + + if 'renderdoccmd' in sys.executable: + args = ['vulkanlayer', '--register', '--system'] + + ctypes.windll.shell32.ShellExecuteW(None, "runas", sys.executable, ' '.join(args), None, 1) + + time.sleep(10) + else: + log.print("Couldn't register vulkan layer properly, might need admin rights") + sys.exit(1) + + reg_needed = rd.NeedVulkanLayerRegistration(layerInfo) + + if reg_needed: log.print("Couldn't register vulkan layer properly, might need admin rights") sys.exit(1) - reg_needed = rd.NeedVulkanLayerRegistration(layerInfo) - - if reg_needed: - log.print("Couldn't register vulkan layer properly, might need admin rights") - sys.exit(1) - os.environ['RENDERDOC_DEMOS_DATA'] = util.get_data_path('demos') testcase.TestCase.set_test_list(fetch_tests()) @@ -362,10 +366,21 @@ def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests duration = datetime.datetime.now(datetime.timezone.utc) - start_time - if len(failedcases) > 0: - logfile = rd.GetLogFile() - if os.path.exists(logfile): - log.inline_file('RenderDoc log', logfile) + if server is not None: + # Connect to the server if running out-of-process + if not server.is_connected(): + server.connect() + + logfile = server.retrieve_latest_server_log(util.get_tmp_dir()) + if logfile is not None and os.path.exists(logfile): + log.inline_file('Replay RenderDoc log', logfile) + + # Do not inline this, as it is usually massive + server.retrieve_comms_log() + + logfile = rd.GetLogFile() + if os.path.exists(logfile): + log.inline_file('{} RenderDoc log'.format("Host" if server is not None else ""), logfile) log.comment("total={} fail={} skip={} time={}".format(len(testcases), len(failedcases), len(skippedcases), int(duration.total_seconds()))) log.header("Tests complete summary: {} passed out of {} run from {} total in {}" @@ -378,6 +393,9 @@ def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests # Print a proper footer if we got here log.rawprint('\n\n\n', with_stdout=False) + if server is not None: + server.shutdown() + rd.ShutdownReplay() if len(failedcases) > 0: @@ -419,6 +437,11 @@ def become_remote_server(): def internal_run_test(test_name): + # In case of out-of-process testing, connect to the server + server = util.get_remote_server() + if server is not None: + server.connect() + testcases = get_tests() log.add_output(util.get_artifact_path("output.log.html")) @@ -440,13 +463,24 @@ def internal_run_test(test_name): except Exception as ex: log.failure(ex) suceeded = False - + logfile = rd.GetLogFile() - if os.path.exists(logfile): - log.inline_file('RenderDoc log', logfile) + if server is not None: + logfile = server.retrieve_latest_test_log(os.path.join(util.get_tmp_dir(), test_name), + None) + + if logfile is not None and os.path.exists(logfile): + log.inline_file('{} RenderDoc log'.format("Test" if server is not None else ""), logfile) log.end_test(test_name, print_footer=False) + if server is not None and server.is_connected(): + server.disconnect() + + # Give some time for the remote to close down, otherwise subsequent tests could fail + # to connect as it's still busy disconnecting + time.sleep(5) + rd.ShutdownReplay() if suceeded: diff --git a/util/test/rdtest/testcase.py b/util/test/rdtest/testcase.py index 0a805e309..271851e6d 100644 --- a/util/test/rdtest/testcase.py +++ b/util/test/rdtest/testcase.py @@ -236,11 +236,18 @@ class TestCase: """ if self.demos_test_name != '': - logfile = os.path.join(util.get_tmp_dir(), 'demos.log') + logfile = os.path.join(util.get_tmp_dir(), util.get_current_test(), 'demos.log') + remote_logfile = logfile + exe = util.get_demos_binary() + if util.get_remote_server() is not None: + remote_logfile = util.get_remote_server().get_temp_path('demos.log') + exe = util.get_remote_server().get_demos_exe() + timeout = self.demos_timeout if timeout is None: timeout = util.get_demos_timeout() - return capture.run_and_capture(util.get_demos_binary(), self.demos_test_name + " --log " + logfile, + return capture.run_and_capture(exe, + self.demos_test_name + " --log " + remote_logfile, self.demos_frame_cap, frame_count=self.demos_frame_count, captures_expected=self.demos_captures_expected, logfile=logfile, opts=self.get_capture_options(), timeout=timeout) @@ -533,7 +540,7 @@ class TestCase: def run(self): self.capture_filename = self.get_capture() - self.check(os.path.exists(self.capture_filename), "Didn't generate capture in make_capture") + self.check(util.target_path_exists(self.capture_filename), "Didn't generate capture in make_capture") log.print("Loading capture") @@ -545,13 +552,16 @@ class TestCase: self.check_capture() if self.controller is not None: - self.controller.Shutdown() + if not util.get_remote_server() is None: + util.get_remote_server().remote.CloseCapture(self.controller) + else: + self.controller.Shutdown() def invoketest(self, debugMode): start_time = self.get_time() self.run() duration = self.get_time() - start_time - log.print("Test ran in {}".format(duration)) + log.print("Test {} ran in {}".format(self.demos_test_name, duration)) self.debugMode = debugMode def get_first_action(self): @@ -798,7 +808,18 @@ class TestCase: return processed + def retrieve_capture(self): + if util.get_remote_server() is None: + return self.capture_filename + + dest = util.get_tmp_path(self.capture_filename.split('/')[-1]) + log.print("Copying remote capture from '{}' to '{}'".format(self.capture_filename, dest)) + util.get_remote_server().remote.CopyCaptureFromRemote(self.capture_filename, dest, None) + return dest + def check_export(self, capture_filename): + capture_filename = self.retrieve_capture() + recomp_path = util.get_tmp_path('recompressed.rdc') conv_zipxml_path = util.get_tmp_path('conv.zip.xml') conv_path = util.get_tmp_path('conv.rdc') diff --git a/util/test/rdtest/util.py b/util/test/rdtest/util.py index 386846fef..2ab59a4e8 100644 --- a/util/test/rdtest/util.py +++ b/util/test/rdtest/util.py @@ -7,8 +7,15 @@ import struct import platform import hashlib import zipfile +import subprocess from typing import Tuple, List from . import png +from rdtest.remoteserver import RemoteServer, AndroidRemoteServer + + +# Android app IDs for the demos +ADRD_DEMO_APP32 = 'renderdoc.org.demos.arm32' +ADRD_DEMO_APP64 = 'renderdoc.org.demos.arm64' def _timestr(): @@ -87,6 +94,24 @@ def set_demos_binary(path: str): _demos_bin = os.path.abspath(path) +def set_remote_server(server: RemoteServer): + global _remote_server + _remote_server = server + + +def get_remote_server(): + return _remote_server + + +def create_adb_device(name: str): + server = AndroidRemoteServer(name) + set_remote_server(server) + + +def get_current_test(): + return _test_name + + def set_demos_timeout(timeout: int): global _demos_timeout _demos_timeout = timeout @@ -142,6 +167,13 @@ def get_tmp_path(name: str): return os.path.join(_temp_dir, _test_name, name) +def get_android_demo_app_name(): + if ADRD_DEMO_APP64 in get_demos_binary(): + return ADRD_DEMO_APP64 + + return ADRD_DEMO_APP32 + + def sanitise_filename(name: str): name = name.replace(_artifact_dir, '') \ .replace(get_tmp_dir(), '') \ @@ -329,3 +361,23 @@ def value_compare(ref, data, eps=FLT_EPSILON): is_eq, diff_amt = value_compare_diff(ref, data, eps) return is_eq + +def run_demo_blocking(args: [str], timeout=100): + """ + Executes the demo application with the given args and returns the stdout. + + :return: stdout, on Android this includes stripping of logcat prefixes + """ + if get_remote_server() is None: + raw = subprocess.run([get_demos_binary()] + args, stdout=subprocess.PIPE).stdout + return str(raw, 'utf-8') + + return get_remote_server().run_demos(args, timeout) + + +def target_path_exists(path: str, timeout=10): + if get_remote_server() is None: + return os.path.exists(path) + + return get_remote_server().path_exists(path) + diff --git a/util/test/run_tests.py b/util/test/run_tests.py index bbd72a742..1e9a72348 100644 --- a/util/test/run_tests.py +++ b/util/test/run_tests.py @@ -36,6 +36,8 @@ parser.add_argument('--temp', default=os.path.join(script_dir, "tmp"), help="The folder to put temporary run data in. Will be completely cleared.", type=str) parser.add_argument('--debugger', help="Enable debugger mode, exceptions are not caught by the framework.", action="store_true") +parser.add_argument('--adb-device', required=False, + help="Use the specified ADB device to run the tests.", type=str) # Internal command, when we fork out to run a test in a separate process parser.add_argument('--internal_run_test', help=argparse.SUPPRESS, type=str, required=False) # Internal command, when we re-run as admin to register vulkan layer @@ -128,6 +130,10 @@ rdtest.set_temp_dir(temp_path) rdtest.set_demos_binary(demos_binary) rdtest.set_demos_timeout(demos_timeout) +if args.adb_device: + rdtest.create_adb_device(args.adb_device) +else: + rdtest.set_remote_server(None) # debugger option implies in-process test running if args.debugger: args.in_process = True