Remote Android and Linux test support part 1

Add remote server support to test framework

util/test/README.md

@@ -42,6 +42,7 @@ Then running the tests means invoking `run_tests.py` with any options you need:
 * `--temp` the path to the temporary working folder, by default `tmp/` here next to the script.
 * `--data-extra` the path to the extra data folder. Some tests may reference captures which can't be committed to the repository here and are distributed separately or added custom by the user. By default refers to `data_extra/` here next to the script.
 * `--demos-binary` the path to the built demos binary.
+* `--adb-device` the ADB device to run the tests on, instead of the host.  If set, --demos-binary is expected to be the demo APK
 
 **NOTE:** When run, the temporary and artifacts folders will be erased.
 

util/test/demos/main.cpp

@@ -395,7 +395,7 @@ int main(int argc, char **argv)
   if(argc >= 2 && (!strcmp(argv[1], "--help") || !strcmp(argv[1], "-h") || !strcmp(argv[1], "-?") ||
                    !strcmp(argv[1], "/help") || !strcmp(argv[1], "/h") || !strcmp(argv[1], "/?")))
   {
-    printf(R"(RenderDoc testing demo program
+    OutputPrint(R"(RenderDoc testing demo program
 
 Usage: %s Test_Name [test_options]
 
@@ -417,9 +417,8 @@ Usage: %s Test_Name [test_options]
                                 environment variable, or else in the data/demos
                                 folder next to the executable.
 )",
-           argc == 0 ? "demos" : argv[0]);
+                argc == 0 ? "demos" : argv[0]);
 
-    fflush(stdout);
     return 1;
   }
 
@@ -434,21 +433,20 @@ Usage: %s Test_Name [test_options]
       if(test.API != prev)
       {
         if(prev != TestAPI::Count)
-          printf("\n\n");
-        printf("======== %s tests ========\n\n", APIName(test.API));
+          OutputPrint("\n\n");
+        OutputPrint("======== %s tests ========\n\n", APIName(test.API));
       }
 
       prev = test.API;
 
-      printf("%s: %s", test.Name, test.IsAvailable() ? "Available" : "Unavailable");
+      OutputPrint("%s: %s", test.Name, test.IsAvailable() ? "Available" : "Unavailable");
 
       if(!test.IsAvailable())
-        printf(" because %s", test.AvailMessage());
+        OutputPrint(" because %s", test.AvailMessage());
 
-      printf("\n\t%s\n\n", test.Description);
+      OutputPrint("\n\t%s\n\n", test.Description);
     }
 
-    fflush(stdout);
     return 1;
   }
 
@@ -459,15 +457,14 @@ Usage: %s Test_Name [test_options]
     check_tests(argc, argv);
 
     // output TSV
-    printf("Name\tAvailable\tAvailMessage\n");
+    OutputPrint("Name\tAvailable\tAvailMessage\n");
 
     for(const TestMetadata &test : tests)
     {
-      printf("%s\t%s\t%s\n", test.Name, test.IsAvailable() ? "True" : "False",
-             test.IsAvailable() ? "Available" : test.AvailMessage());
+      OutputPrint("%s\t%s\t%s\n", test.Name, test.IsAvailable() ? "True" : "False",
+                  test.IsAvailable() ? "Available" : test.AvailMessage());
     }
 
-    fflush(stdout);
     return 1;
   }
 
@@ -697,7 +694,6 @@ Usage: %s Test_Name [test_options]
   }
 
   TEST_ERROR("%s is not a known test", argv[1]);
-
   return 2;
 }
 
@@ -745,7 +741,7 @@ int WINAPI wWinMain(_In_ HINSTANCE hInst, _In_opt_ HINSTANCE hPrevInstance, _In_
 struct android_app *android_state;
 pthread_t cmdthread_handle = 0;
 
-#define ANDROID_LOG(...) __android_log_print(ANDROID_LOG_INFO, "rd_demos", __VA_ARGS__);
+#define ANDROID_LOG(...) __android_log_print(ANDROID_LOG_DEBUG, "rd_demos", __VA_ARGS__);
 
 std::vector<std::string> getArgs()
 {

util/test/demos/test_common.cpp

@@ -190,6 +190,33 @@ void DebugPrint(const char *fmt, ...)
   OutputDebugStringA(printBuf);
 #endif
 
+#if defined(ANDROID)
+  __android_log_print(ANDROID_LOG_DEBUG, "rd_demos", "%s", printBuf);
+#endif
+}
+
+void OutputPrint(const char *fmt, ...)
+{
+  va_list args;
+  va_start(args, fmt);
+
+  vsnprintf(printBuf, 4095, fmt, args);
+
+  va_end(args);
+
+  fputs(printBuf, stdout);
+  fflush(stdout);
+
+  if(logFile)
+  {
+    fputs(printBuf, logFile);
+    fflush(logFile);
+  }
+
+#if defined(WIN32)
+  OutputDebugStringA(printBuf);
+#endif
+
 #if defined(ANDROID)
   __android_log_print(ANDROID_LOG_INFO, "rd_demos", "%s", printBuf);
 #endif

util/test/demos/test_common.h

@@ -337,6 +337,7 @@ std::string trim(const std::string &str);
 
 void SetDebugLogEnabled(bool enabled);
 void DebugPrint(const char *fmt, ...);
+void OutputPrint(const char *fmt, ...);
 
 #define TEST_ASSERT(cond, fmt, ...)                                                                 \
   do                                                                                                \

util/test/rdtest/analyse.py

@@ -1,6 +1,7 @@
 import struct
 from typing import List
 import renderdoc
+from . import util
 
 # Alias for convenience - we need to import as-is so types don't get confused
 rd = renderdoc
@@ -24,27 +25,39 @@ def open_capture(filename="", cap: rd.CaptureFile=None, opts: rd.ReplayOptions=N
     own_cap = False
     api = "Unknown"
 
-    if cap is None:
-        own_cap = True
+    result = None
+    controller = None
+    if util.get_remote_server() is None:
+        if cap is None:
+            own_cap = True
 
-        cap = rd.OpenCaptureFile()
+            cap = rd.OpenCaptureFile()
 
-        # Open a particular file
-        result = cap.OpenFile(filename, '', None)
+            # Open a particular file
+            result = cap.OpenFile(filename, '', None)
 
-        # Make sure the file opened successfully
-        if result != rd.ResultCode.Succeeded:
-            cap.Shutdown()
-            raise RuntimeError("Couldn't open '{}': {}".format(filename, str(result)))
+            # Make sure the file opened successfully
+            if result != rd.ResultCode.Succeeded:
+                cap.Shutdown()
+                raise RuntimeError("Couldn't open '{}': {}".format(filename, str(result)))
 
-        api = cap.DriverName()
+            api = cap.DriverName()
 
-        # Make sure we can replay
-        if not cap.LocalReplaySupport():
-            cap.Shutdown()
-            raise RuntimeError("{} capture cannot be replayed".format(api))
+            # Make sure we can replay
+            if not cap.LocalReplaySupport():
+                cap.Shutdown()
+                raise RuntimeError("{} capture cannot be replayed".format(api))
 
-    result, controller = cap.OpenCapture(opts, None)
+        result, controller = cap.OpenCapture(opts, None)
+    else:
+        if not cap is None:
+            raise ValueError("Cannot call analyse.open_capture() with capture handle for remote {}"
+                        .format(util.get_remote_server().remote))
+
+        result, controller = util.get_remote_server().remote.OpenCapture(rd.RemoteServer.NoPreference,
+                                                                   filename, opts, None)
+        if result == rd.ResultCode.Succeeded:
+            api = util.get_remote_server().remote.DriverName()
 
     if own_cap:
         cap.Shutdown()

util/test/rdtest/capture.py

@@ -5,6 +5,7 @@ import time
 import renderdoc as rd
 from . import util
 from .logging import log
+from time import sleep
 
 
 class TargetControl():
@@ -134,10 +135,14 @@ def run_executable(exe: str, cmdline: str,
 
     wait_for_exit = False
 
-    log.print("Running exe:'{}' cmd:'{}' in dir:'{}' with env:'{}'".format(exe, cmdline, workdir, envmods))
-
     # Execute the test program
-    res = rd.ExecuteAndInject(exe, workdir, cmdline, envmods, cappath, opts, wait_for_exit)
+    server = util.get_remote_server()
+    res = None
+    if server is None:
+        log.print("Running exe:'{}' cmd:'{}' in dir:'{}' with env:'{}'".format(exe, cmdline, workdir, envmods))
+        res = rd.ExecuteAndInject(exe, workdir, cmdline, envmods, cappath, opts, wait_for_exit)
+    else:
+        res = server.inject_and_run_exe(cmdline, envmods, opts)
 
     if res.result != rd.ResultCode.Succeeded:
         raise RuntimeError("Couldn't launch program: {}".format(str(res.result)))
@@ -171,7 +176,20 @@ def run_and_capture(exe: str, cmdline: str, frame: int, *, frame_count=1, captur
     if captures_expected is None:
         captures_expected = frame_count
 
-    control = TargetControl(run_executable(exe, cmdline, cappath=util.get_tmp_path(capture_name), opts=opts), timeout=timeout)
+    host = "localhost"
+    username = "testrunner"
+    cappath = ""
+
+    server = util.get_remote_server()
+    if server is not None:
+        cappath = server.get_temp_path(capture_name)
+        host = server.get_hostname()
+        username = server.get_username()
+    else:
+        cappath = util.get_tmp_path(capture_name)
+
+    control = TargetControl(run_executable(exe, cmdline, cappath=cappath, opts=opts),
+                            host=host, username=username, timeout=timeout)
 
     log.print("Queuing capture of frame {}..{} with timeout of {}".format(frame, frame+frame_count, "default" if timeout is None else timeout))
 
@@ -183,6 +201,15 @@ def run_and_capture(exe: str, cmdline: str, frame: int, *, frame_count=1, captur
     control.run(keep_running=lambda x: len(x.captures()) < captures_expected)
 
     captures = control.captures()
+    log.print(f'Retrieved {len(captures)} captures')
+
+    # Retrieve the demo logfile from the remote device
+    if server is not None:
+        remote_logfile = server.get_temp_path('demos.log')
+        if server.path_exists(remote_logfile):
+            log.print("Copying remote demo log from '{}' to '{}'".format(server.get_temp_path('demos.log'), logfile))
+            os.makedirs(os.path.dirname(logfile), exist_ok=True)
+            server.remote.CopyCaptureFromRemote(server.get_temp_path('demos.log'), logfile, None)
 
     if logfile is not None and os.path.exists(logfile):
         log.inline_file('Process output', logfile, with_stdout=True)

util/test/rdtest/remoteserver.py

@@ -0,0 +1,335 @@
+
+import sys
+import subprocess
+import renderdoc as rd
+from . import util
+from .logging import log
+from pathlib import Path
+import os
+import re
+
+from time import sleep
+from abc import ABC, abstractmethod
+
+
+class RemoteServer(ABC):
+    def __init__(self) -> None:
+        super().__init__()
+        self.device = None
+        self.remote = None
+
+    @abstractmethod
+    def init(self, in_process):
+        pass
+
+    @abstractmethod
+    def connect(self):
+        pass
+
+    @abstractmethod
+    def disconnect(self):
+        pass
+
+    @abstractmethod
+    def shutdown(self):
+        pass
+
+    @abstractmethod
+    def is_connected(self):
+        pass
+
+    @abstractmethod
+    def get_temp_path(self, name, timeout):
+        pass
+
+    @abstractmethod
+    def get_renderdoc_path(self):
+        pass
+
+    @abstractmethod
+    def path_exists(self, path, timeout):
+        pass
+
+    @abstractmethod
+    def run_demos(self, args, timeout):
+        pass
+
+    @abstractmethod
+    def inject_and_run_exe(self, cmdline, envmods, opts):
+        pass
+
+    @abstractmethod
+    def get_demos_exe(self):
+        pass
+
+    @abstractmethod
+    def get_hostname(self):
+        pass
+
+    @abstractmethod
+    def get_username(self):
+        pass
+
+    @abstractmethod
+    def retrieve_latest_test_log(self, dst, timeout):
+        pass
+
+    @abstractmethod
+    def retrieve_latest_server_log(self, dst, timeout):
+        pass
+
+    @abstractmethod
+    def retrieve_comms_log(self, timeout):
+        pass
+
+
+class AndroidRemoteServer(RemoteServer):
+    # Android app IDs for the server
+    ADRD_SERVER_APP64 = 'org.renderdoc.renderdoccmd.arm64'
+    CONNECTION_RETRY_COUNT = 3
+
+    def __init__(self, device) -> None:
+        super().__init__()
+        if device is None:
+            raise RuntimeError('Android target specified, but no device given')
+        self.device = device
+        self.remote = None
+        self._base_path = ''
+
+    def init(self, in_process):        
+        # Remove any existing Vulkan layers
+        subprocess.run(['adb', '-s', self.device,
+                        'shell', 'settings', 'delete', 'global', 'gpu_debug_layers'], check=False)
+
+        remote = self.connect()
+        log.print(f'Supported drivers: {remote.RemoteSupportedReplays()}')
+
+        # Install the demo APK
+        subprocess.run(['adb', '-s', self.device,
+                        'install', '-g', util.get_demos_binary()], check=True)
+
+        # Remove any stale logs and captures from previous runs
+        rm_glob = self._base_path + '/*'
+        subprocess.run(['adb', '-s', self.device, 'shell',
+                       'rm', '-rf', rm_glob], check=True)
+
+        # Close the connection if the tests are forked as each test will create their own
+        # connection
+        if not in_process:
+            self.disconnect()
+
+    def connect(self):
+        log.print("Connecting to remote server...")
+
+        protocols = rd.GetSupportedDeviceProtocols()
+        if not 'adb' in protocols:
+            log.print('ADB requested but not an available device protocol')
+            sys.exit(1)
+
+        protocol = rd.GetDeviceProtocolController('adb')
+        if not self.device in protocol.GetDevices():
+            log.print(f'ADB device {self.device} requested but not discovered')
+            sys.exit(1)
+
+        if not protocol.IsSupported(self.device):
+            log.print(f'ADB device {self.device} is not supported')
+            sys.exit(1)
+
+        url = f'{protocol.GetProtocolName()}://{self.device}'
+        result, remote = rd.CreateRemoteServerConnection(url)
+        if result == rd.ResultCode.NetworkIOFailed and protocol is not None:
+            log.print("Couldn't connect to remote server, trying to start it")
+
+            result = protocol.StartRemoteServer(url)
+            if result != rd.ResultCode.Succeeded:
+                log.print(
+                    f"Couldn't launch remote server, got error {str(result)}")
+                sys.exit(1)
+
+            # Try to connect again!
+            result, remote = rd.CreateRemoteServerConnection(url)
+
+        # Retry a few times
+        if result != rd.ResultCode.Succeeded:
+            for i in range(1, self.CONNECTION_RETRY_COUNT + 1):
+                result, remote = rd.CreateRemoteServerConnection(url)
+                if result == rd.ResultCode.Succeeded:
+                    break
+                
+                log.print(
+                    f"Couldn't connect to remote server on attempt #{i}, got error {str(result)}")
+                if i == self.CONNECTION_RETRY_COUNT:
+                    sys.exit(1)
+
+        # Calculate the remote base path
+        base = '/sdcard/Android/'
+        output = subprocess.run(['adb', '-s', self.device, 'shell', 'getprop', 'ro.build.version.sdk'],
+                                stdout=subprocess.PIPE, timeout=10, check=True).stdout
+        api_version = int(str(output, 'utf-8').strip())
+        if api_version >= 30:
+            base += 'media/'
+        else:
+            base += 'data/'
+
+        self._data_path = base
+        self._base_path = base + util.get_android_demo_app_name() + '/files/'
+        self.remote = remote
+
+        log.print("Connected!")
+        return remote
+
+    def disconnect(self):
+        self.remote.ShutdownConnection()
+        self.remote = None
+
+    def shutdown(self):
+        # If we running over ADB, close down the server.  This will involve first establishing a
+        # connection if the tests have been running out-of-process
+        log.print('Shutting down server...')
+
+        # Kill the demo app first
+        subprocess.run(['adb', '-s', self.device, 'shell', 'am', 'force-stop',
+                        util.get_android_demo_app_name()])
+
+        if self.remote is None:
+            protocol = rd.GetDeviceProtocolController('adb')
+            url = f'{protocol.GetProtocolName()}://{self.device}'
+            result, self.remote = rd.CreateRemoteServerConnection(url)
+            if result != rd.ResultCode.Succeeded:
+                log.print(
+                    f"Couldn't connect to remote server for shutdown, got error {str(result)}")
+                return
+
+        self.remote.ShutdownServerAndConnection()
+        self.remote = None
+
+    def is_connected(self) -> bool:
+        return self.remote is not None
+
+    def get_temp_path(self, name="", timeout=20):
+        subprocess.run(['adb', '-s', self.device, 'shell', 'mkdir', '-p',
+                        self._base_path + '/' + util.get_current_test()],
+                       timeout=timeout,
+                       check=True)
+        return self._base_path + util.get_current_test() + '/' + name
+
+    def get_renderdoc_path(self):
+        return self._data_path + '/' + AndroidRemoteServer.ADRD_SERVER_APP64 + '/files/RenderDoc/'
+
+    def run_demos(self, args: [str], timeout=10):
+        raw = subprocess.run(['adb', '-s', self.device, 'shell', 'echo', '$EPOCHREALTIME'],
+                             check=True, stdout=subprocess.PIPE, timeout=timeout).stdout
+        ts = str(raw, 'utf-8').strip()
+        # Run the command, blocking
+        proc = subprocess.run(['adb', '-s', self.device,
+                               'shell', 'am', 'start', '-W', '-n', f'{util.get_android_demo_app_name()}/.Loader',
+                               '-e', 'demos', 'RenderDoc', '-e', 'rd_demos'] + args,
+                              check=True, stdout=subprocess.DEVNULL)
+        # Extract the log data
+        raw = subprocess.run(['adb', '-s', self.device, 'shell',
+                              'logcat', '-b', 'main', '-v', 'brief', '-T', ts, '-d', 'rd_demos:I', '*:S'],
+                             check=True, stdout=subprocess.PIPE, timeout=timeout).stdout
+        output = str(raw, 'utf-8')
+        # Remove the per-line prefix
+        output = output.splitlines()
+        result = ""
+        for line in output:
+            pos = line.find('): ')
+            if not line.startswith('I/rd_demos(') or pos == -1:
+                continue
+
+            result += line[pos+3:] + '\n'
+
+        return result
+
+    def path_exists(self, path, timeout=10):
+        raw = subprocess.run(['adb', '-s', self.device, 'shell', f'ls {path} >> /dev/null'],
+                             stderr=subprocess.PIPE, timeout=timeout).stderr
+        raw = str(raw, 'utf-8').strip()
+
+        return len(raw) == 0
+
+    def inject_and_run_exe(self, cmdline, envmods, opts):
+        package_and_activity = f"{util.get_android_demo_app_name()}/.Loader"
+        args = "-e demos RenderDoc -e rd_demos \'\"" + cmdline + "\"\'"
+
+        log.print("Running package:'{}' cmd:'{}' with env:'{}'".format(
+            package_and_activity, cmdline, envmods))
+        res = util.get_remote_server().remote.ExecuteAndInject(
+            package_and_activity, "", args, envmods, opts)
+
+        if res.result != rd.ResultCode.Succeeded:
+            raise RuntimeError(
+                "Couldn't launch program: {}".format(str(res.result)))
+
+        return res
+
+    def get_demos_exe(self):
+        return util.get_android_demo_app_name()
+
+    def get_hostname(self):
+        return f"adb://{self.device}"
+
+    def get_username(self):
+        return "testrunner"
+
+    def retrieve_latest_test_log(self, dst, timeout=10):
+        if not self.is_connected():
+            return None
+
+        src = self._base_path + '/RenderDoc'
+        raw = subprocess.run(['adb', '-s', self.device, 'shell', f'cd {src}; ls -t RenderDoc_* | head -1'],
+                             check=True, stdout=subprocess.PIPE, timeout=timeout).stdout
+        latestlog = str(raw, 'utf-8').strip()
+        if not latestlog:
+            log.print(f"Cannot find latest remote log from '{src}'")
+            return None
+
+        os.makedirs(dst, exist_ok=True)
+
+        dst = os.path.join(dst, latestlog)
+        src = os.path.join(src, latestlog)
+        log.print(f"Copying remote test log from '{src}' to '{dst}'")
+        self.remote.CopyCaptureFromRemote(src, dst, None)
+
+        return dst
+
+    def retrieve_latest_server_log(self, dst, timeout=10):
+        if not self.is_connected():
+            return None
+
+        src = self.get_renderdoc_path()
+        raw = subprocess.run(['adb', '-s', self.device, 'shell', f'cd {src}; ls -t RenderDoc_* | head -1'],
+                             check=True, stdout=subprocess.PIPE, timeout=timeout).stdout
+        latestlog = str(raw, 'utf-8').strip()
+        if not latestlog:
+            log.print(f"Cannot find latest remote log from '{src}'")
+            return None
+
+        os.makedirs(dst, exist_ok=True)
+
+        dst = os.path.join(dst, latestlog)
+        src = os.path.join(src, latestlog)
+        log.print(f"Copying remote server log from '{src}' to '{dst}'")
+        self.remote.CopyCaptureFromRemote(src, dst, None)
+
+        return dst
+
+    def retrieve_comms_log(self, timeout=10):
+        if not self.is_connected():
+            return None
+
+        src = self.get_renderdoc_path() + "RemoteServer_Server.log"
+        if not self.path_exists(src):
+            log.print(f"Cannot find server comms log '{src}'")
+            return None
+
+        os.makedirs(util.get_tmp_dir(), exist_ok=True)
+
+        dst = os.path.join(util.get_tmp_dir(), 'RenderDoc_Server.log')
+        log.print("Copying remote server comms log from '{}' to '{}'".format(src, dst))
+        self.remote.CopyCaptureFromRemote(src, dst, None)
+
+        return dst
+
+

util/test/rdtest/runner.py

@@ -14,6 +14,7 @@ from . import util
 from . import testcase
 from .logging import log
 from pathlib import Path
+from rdtest.remoteserver import RemoteServer
 
 
 def get_tests():
@@ -176,11 +177,11 @@ def _run_test(testclass, runner_timeout, failedcases: list):
                            .format(test_run.returncode))
 
 
-def fetch_tests():
-    output = subprocess.run([util.get_demos_binary(), '--list-raw'], stdout=subprocess.PIPE).stdout
-
-    # Skip the header, grab all the remaining lines
-    tests = str(output, 'utf-8').splitlines()[1:]
+def fetch_tests():  
+    output = util.run_demo_blocking(['--list-raw']).splitlines()
+    
+    # Skip to just past the header, grab all the remaining lines
+    tests = output[output.index("Name\tAvailable\tAvailMessage")+1:]
 
     # Split the TSV values and store
     split_tests = [ test.split('\t') for test in tests ]
@@ -193,6 +194,10 @@ def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests
 
     rd.InitialiseReplay(rd.GlobalEnvironment(), [])
 
+    server: RemoteServer = util.get_remote_server()
+    if server is not None:
+        server.init(in_process)
+
     # On windows, disable error reporting
     if 'windll' in dir(ctypes):
         ctypes.windll.kernel32.SetErrorMode(1 | 2)  # SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX
@@ -222,64 +227,63 @@ def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests
     log.header("On {}".format(platform.platform()))
 
     log.comment("plat={} git={}".format(platform.platform(), rd.GetCommitHash()))
-
-    driver = ""
-
-    for api in rd.GraphicsAPI:
-        v = rd.GetDriverInformation(api)
-        log.print("{} driver: {} {}".format(str(api), str(v.vendor), v.version))
-
-        # Take the first version number we get, but prefer GL as it's universally available and
-        # Produces a nice version number & device combination
-        if (api == rd.GraphicsAPI.OpenGL or driver == "") and v.vendor != rd.GPUVendor.Unknown:
-            driver = v.version
-
-    log.comment("driver={}".format(driver))
-
     log.print("Demos running from {}".format(util.get_demos_binary()))
 
-    layerInfo = rd.VulkanLayerRegistrationInfo()
-    if rd.NeedVulkanLayerRegistration(layerInfo):
-        log.print("Vulkan layer needs to be registered: {}".format(str(layerInfo.flags)))
-        log.print("My JSONs: {}, Other JSONs: {}".format(layerInfo.myJSONs, layerInfo.otherJSONs))
+    if server is None:
+        driver = ""
+        for api in rd.GraphicsAPI:
+            v = rd.GetDriverInformation(api)
+            log.print("{} driver: {} {}".format(str(api), str(v.vendor), v.version))
 
-        # Update the layer registration without doing anything special first - if running automated we might have
-        # granted user-writable permissions to the system files needed to update. If possible we register at user
-        # level.
-        if layerInfo.flags & rd.VulkanLayerFlags.NeedElevation:
-            rd.UpdateVulkanLayerRegistration(True)
-        else:
-            rd.UpdateVulkanLayerRegistration(False)
+            # Take the first version number we get, but prefer GL as it's universally available and
+            # Produces a nice version number & device combination
+            if (api == rd.GraphicsAPI.OpenGL or driver == "") and v.vendor != rd.GPUVendor.Unknown:
+                driver = v.version
 
-        # Check if it succeeded
-        reg_needed = rd.NeedVulkanLayerRegistration(layerInfo)
+        log.comment("driver={}".format(driver))
 
-        if reg_needed:
-            if plat == 'win32':
-                # On windows, try to elevate. This will mean a UAC prompt
-                args = sys.argv.copy()
-                args.append("--internal_vulkan_register")
+        layerInfo = rd.VulkanLayerRegistrationInfo()
+        if rd.NeedVulkanLayerRegistration(layerInfo):
+            log.print("Vulkan layer needs to be registered: {}".format(str(layerInfo.flags)))
+            log.print("My JSONs: {}, Other JSONs: {}".format(layerInfo.myJSONs, layerInfo.otherJSONs))
 
-                for i in range(len(args)):
-                    if os.path.exists(args[i]):
-                        args[i] = str(Path(args[i]).resolve())
-
-                if 'renderdoccmd' in sys.executable:
-                    args = ['vulkanlayer', '--register', '--system']
-
-                ctypes.windll.shell32.ShellExecuteW(None, "runas", sys.executable, ' '.join(args), None, 1)
-
-                time.sleep(10)
+            # Update the layer registration without doing anything special first - if running automated we might have
+            # granted user-writable permissions to the system files needed to update. If possible we register at user
+            # level.
+            if layerInfo.flags & rd.VulkanLayerFlags.NeedElevation:
+                rd.UpdateVulkanLayerRegistration(True)
             else:
+                rd.UpdateVulkanLayerRegistration(False)
+
+            # Check if it succeeded
+            reg_needed = rd.NeedVulkanLayerRegistration(layerInfo)
+
+            if reg_needed:
+                if plat == 'win32':
+                    # On windows, try to elevate. This will mean a UAC prompt
+                    args = sys.argv.copy()
+                    args.append("--internal_vulkan_register")
+
+                    for i in range(len(args)):
+                        if os.path.exists(args[i]):
+                            args[i] = str(Path(args[i]).resolve())
+
+                    if 'renderdoccmd' in sys.executable:
+                        args = ['vulkanlayer', '--register', '--system']
+
+                    ctypes.windll.shell32.ShellExecuteW(None, "runas", sys.executable, ' '.join(args), None, 1)
+
+                    time.sleep(10)
+                else:
+                    log.print("Couldn't register vulkan layer properly, might need admin rights")
+                    sys.exit(1)
+
+            reg_needed = rd.NeedVulkanLayerRegistration(layerInfo)
+
+            if reg_needed:
                 log.print("Couldn't register vulkan layer properly, might need admin rights")
                 sys.exit(1)
 
-        reg_needed = rd.NeedVulkanLayerRegistration(layerInfo)
-
-        if reg_needed:
-            log.print("Couldn't register vulkan layer properly, might need admin rights")
-            sys.exit(1)
-
     os.environ['RENDERDOC_DEMOS_DATA'] = util.get_data_path('demos')
 
     testcase.TestCase.set_test_list(fetch_tests())
@@ -362,10 +366,21 @@ def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests
 
     duration = datetime.datetime.now(datetime.timezone.utc) - start_time
 
-    if len(failedcases) > 0:
-        logfile = rd.GetLogFile()
-        if os.path.exists(logfile):
-            log.inline_file('RenderDoc log', logfile)
+    if server is not None:
+        # Connect to the server if running out-of-process
+        if not server.is_connected():
+            server.connect()
+
+        logfile = server.retrieve_latest_server_log(util.get_tmp_dir())
+        if logfile is not None and os.path.exists(logfile):
+            log.inline_file('Replay RenderDoc log', logfile)
+
+        # Do not inline this, as it is usually massive
+        server.retrieve_comms_log()
+
+    logfile = rd.GetLogFile()
+    if os.path.exists(logfile):
+        log.inline_file('{} RenderDoc log'.format("Host" if server is not None else ""), logfile)
 
     log.comment("total={} fail={} skip={} time={}".format(len(testcases), len(failedcases), len(skippedcases), int(duration.total_seconds())))
     log.header("Tests complete summary: {} passed out of {} run from {} total in {}"
@@ -378,6 +393,9 @@ def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests
     # Print a proper footer if we got here
     log.rawprint('\n\n\n</script>', with_stdout=False)
 
+    if server is not None:
+        server.shutdown()
+        
     rd.ShutdownReplay()
 
     if len(failedcases) > 0:
@@ -419,6 +437,11 @@ def become_remote_server():
 
 
 def internal_run_test(test_name):
+    # In case of out-of-process testing, connect to the server
+    server = util.get_remote_server()
+    if server is not None:
+        server.connect()
+
     testcases = get_tests()
 
     log.add_output(util.get_artifact_path("output.log.html"))
@@ -440,13 +463,24 @@ def internal_run_test(test_name):
             except Exception as ex:
                 log.failure(ex)
                 suceeded = False
-                
+
             logfile = rd.GetLogFile()
-            if os.path.exists(logfile):
-                log.inline_file('RenderDoc log', logfile)
+            if server is not None:
+                logfile = server.retrieve_latest_test_log(os.path.join(util.get_tmp_dir(), test_name),
+                                                          None)
+
+            if logfile is not None and os.path.exists(logfile):
+                log.inline_file('{} RenderDoc log'.format("Test" if server is not None else ""), logfile)
 
             log.end_test(test_name, print_footer=False)
 
+            if server is not None and server.is_connected():
+                server.disconnect()
+
+                # Give some time for the remote to close down, otherwise subsequent tests could fail
+                # to connect as it's still busy disconnecting
+                time.sleep(5)
+
             rd.ShutdownReplay()
 
             if suceeded:

util/test/rdtest/testcase.py

@@ -236,11 +236,18 @@ class TestCase:
         """
 
         if self.demos_test_name != '':
-            logfile = os.path.join(util.get_tmp_dir(), 'demos.log')
+            logfile = os.path.join(util.get_tmp_dir(), util.get_current_test(), 'demos.log')
+            remote_logfile = logfile
+            exe = util.get_demos_binary()
+            if util.get_remote_server() is not None:
+                remote_logfile = util.get_remote_server().get_temp_path('demos.log')
+                exe = util.get_remote_server().get_demos_exe()
+
             timeout = self.demos_timeout
             if timeout is None:
                 timeout = util.get_demos_timeout()
-            return capture.run_and_capture(util.get_demos_binary(), self.demos_test_name + " --log " + logfile,
+            return capture.run_and_capture(exe,
+                                           self.demos_test_name + " --log " + remote_logfile,
                                            self.demos_frame_cap, frame_count=self.demos_frame_count,
                                            captures_expected=self.demos_captures_expected, logfile=logfile,
                                            opts=self.get_capture_options(), timeout=timeout)
@@ -533,7 +540,7 @@ class TestCase:
     def run(self):
         self.capture_filename = self.get_capture()
 
-        self.check(os.path.exists(self.capture_filename), "Didn't generate capture in make_capture")
+        self.check(util.target_path_exists(self.capture_filename), "Didn't generate capture in make_capture")
 
         log.print("Loading capture")
 
@@ -545,13 +552,16 @@ class TestCase:
         self.check_capture()
 
         if self.controller is not None:
-            self.controller.Shutdown()
+            if not util.get_remote_server() is None:
+                util.get_remote_server().remote.CloseCapture(self.controller)
+            else:
+                self.controller.Shutdown()
 
     def invoketest(self, debugMode):
         start_time = self.get_time()
         self.run()
         duration = self.get_time() - start_time
-        log.print("Test ran in {}".format(duration))
+        log.print("Test {} ran in {}".format(self.demos_test_name, duration))
         self.debugMode = debugMode
 
     def get_first_action(self):
@@ -798,7 +808,18 @@ class TestCase:
 
         return processed
 
+    def retrieve_capture(self):
+        if util.get_remote_server() is None:
+            return self.capture_filename
+
+        dest = util.get_tmp_path(self.capture_filename.split('/')[-1])
+        log.print("Copying remote capture from '{}' to '{}'".format(self.capture_filename, dest))
+        util.get_remote_server().remote.CopyCaptureFromRemote(self.capture_filename, dest, None)
+        return dest
+
     def check_export(self, capture_filename):
+        capture_filename = self.retrieve_capture()
+
         recomp_path = util.get_tmp_path('recompressed.rdc')
         conv_zipxml_path = util.get_tmp_path('conv.zip.xml')
         conv_path = util.get_tmp_path('conv.rdc')

util/test/rdtest/util.py

@@ -7,8 +7,15 @@ import struct
 import platform
 import hashlib
 import zipfile
+import subprocess
 from typing import Tuple, List
 from . import png
+from rdtest.remoteserver import RemoteServer, AndroidRemoteServer 
+
+
+# Android app IDs for the demos
+ADRD_DEMO_APP32 = 'renderdoc.org.demos.arm32'
+ADRD_DEMO_APP64 = 'renderdoc.org.demos.arm64'
 
 
 def _timestr():
@@ -87,6 +94,24 @@ def set_demos_binary(path: str):
         _demos_bin = os.path.abspath(path)
 
 
+def set_remote_server(server: RemoteServer):
+    global _remote_server
+    _remote_server = server
+
+
+def get_remote_server():
+    return _remote_server
+
+
+def create_adb_device(name: str):
+    server = AndroidRemoteServer(name)
+    set_remote_server(server)
+
+
+def get_current_test():
+    return _test_name
+
+
 def set_demos_timeout(timeout: int):
     global _demos_timeout
     _demos_timeout = timeout
@@ -142,6 +167,13 @@ def get_tmp_path(name: str):
     return os.path.join(_temp_dir, _test_name, name)
 
 
+def get_android_demo_app_name():
+    if ADRD_DEMO_APP64 in get_demos_binary():
+        return ADRD_DEMO_APP64
+
+    return ADRD_DEMO_APP32
+
+
 def sanitise_filename(name: str):
     name = name.replace(_artifact_dir, '') \
                .replace(get_tmp_dir(), '') \
@@ -329,3 +361,23 @@ def value_compare(ref, data, eps=FLT_EPSILON):
     is_eq, diff_amt = value_compare_diff(ref, data, eps)
     return is_eq
 
+
+def run_demo_blocking(args: [str], timeout=100):
+    """
+    Executes the demo application with the given args and returns the stdout.
+
+    :return: stdout, on Android this includes stripping of logcat prefixes
+    """
+    if get_remote_server() is None:
+        raw = subprocess.run([get_demos_binary()] + args, stdout=subprocess.PIPE).stdout
+        return str(raw, 'utf-8')
+
+    return get_remote_server().run_demos(args, timeout)
+
+
+def target_path_exists(path: str, timeout=10):
+    if get_remote_server() is None:
+        return os.path.exists(path)
+    
+    return get_remote_server().path_exists(path)
+

util/test/run_tests.py

@@ -36,6 +36,8 @@ parser.add_argument('--temp', default=os.path.join(script_dir, "tmp"),
                     help="The folder to put temporary run data in. Will be completely cleared.", type=str)
 parser.add_argument('--debugger',
                     help="Enable debugger mode, exceptions are not caught by the framework.", action="store_true")
+parser.add_argument('--adb-device', required=False,
+                    help="Use the specified ADB device to run the tests.", type=str)
 # Internal command, when we fork out to run a test in a separate process
 parser.add_argument('--internal_run_test', help=argparse.SUPPRESS, type=str, required=False)
 # Internal command, when we re-run as admin to register vulkan layer
@@ -128,6 +130,10 @@ rdtest.set_temp_dir(temp_path)
 rdtest.set_demos_binary(demos_binary)
 rdtest.set_demos_timeout(demos_timeout)
 
+if args.adb_device:
+    rdtest.create_adb_device(args.adb_device)
+else:
+    rdtest.set_remote_server(None)
 # debugger option implies in-process test running
 if args.debugger:
     args.in_process = True