Add fallback on android if gpu debug layers are broken or inaccessible

renderdoc/android/android.cpp

@@ -1163,6 +1163,7 @@ ExecuteResult AndroidRemoteServer::ExecuteAndInject(const char *a, const char *w
     Android::adbExecCommand(m_deviceID, StringFormat::Fmt("forward --remove tcp:%i", jdwpPort));
     // force stop the package if it was running before
     Android::adbExecCommand(m_deviceID, "shell am force-stop " + processName);
+    Android::adbExecCommand(m_deviceID, "shell setprop debug.vulkan.layers :", ".", true);
 
     bool hookWithJDWP = true;
 
@@ -1205,14 +1206,53 @@ ExecuteResult AndroidRemoteServer::ExecuteAndInject(const char *a, const char *w
           m_deviceID, "shell settings put global gpu_debug_layers " RENDERDOC_VULKAN_LAYER_NAME);
       Android::adbExecCommand(
           m_deviceID, "shell settings put global gpu_debug_layers_gles " RENDERDOC_ANDROID_LIBRARY);
+
+      // don't ignore the layers by default, only if we encounter an error
+      Android::adbExecCommand(m_deviceID, "shell setprop debug.rdoc.IGNORE_LAYERS 0");
+
+      Process::ProcessResult check =
+          Android::adbExecCommand(m_deviceID, "shell settings list global");
+
+      // check both since we don't know which one it will come out in
+      rdcstr inString = check.strStdout + check.strStderror;
+
+      // remove all whitespace. Our package and layer doesn't contain spaces, and the user's package
+      // name can't contain spaces. This makes what we're searching for less subject to change (e.g.
+      // if some adb versions print 'setting = value' instead of 'setting=value'
+      // This will even work if there are new lines
+      rdcstr checkString;
+      checkString.reserve(inString.size());
+      for(const char &c : inString)
+      {
+        if(c == ' ' || c == '\t' || c == '\r' || c == '\n')
+          continue;
+
+        checkString.push_back(c);
+      }
+
+      if(!checkString.contains("enable_gpu_debug_layers=1") ||
+         !checkString.contains("gpu_debug_app=" + packageName) ||
+         !checkString.contains("gpu_debug_layer_app=" + layerPackage) ||
+         !checkString.contains("gpu_debug_layers=" RENDERDOC_VULKAN_LAYER_NAME) ||
+         !checkString.contains("gpu_debug_layers_gles=" RENDERDOC_ANDROID_LIBRARY))
+      {
+        RDCERR(
+            "Couldn't verify that debug settings are set:\n%s"
+            "Do you have a strange device that requires extra setup?\n"
+            "E.g. Xiaomi requires a developer account and \"USB debugging (Security Settings)\"\n",
+            inString.c_str());
+        hookWithJDWP = true;
+
+        // need to tell the hooks to ignore the fact that layers are present because they're not
+        // working.
+        Android::adbExecCommand(m_deviceID, "shell setprop debug.rdoc.IGNORE_LAYERS 1");
+      }
     }
-    else
+
+    if(hookWithJDWP)
     {
       RDCLOG("Using pre-Android 10 Vulkan layering and JDWP injection");
 
-      // use JDWP hooking to inject our library for GLES
-      hookWithJDWP = true;
-
       // enable the vulkan layer (will only be used by vulkan programs)
       Android::adbExecCommand(m_deviceID,
                               "shell setprop debug.vulkan.layers " RENDERDOC_VULKAN_LAYER_NAME);

renderdoc/driver/gl/egl_hooks.cpp

@@ -949,6 +949,13 @@ bool ShouldHookEGL()
     return true;
   }
 
+  const char *ignore_layers = Process::GetEnvVariable("IGNORE_LAYERS");
+
+  // if we set IGNORE_LAYERS externally that means the layers are broken or can't be configured, so
+  // hook EGL in spite of the layers being present
+  if(ignore_layers && ignore_layers[0] == '1')
+    return true;
+
   const char *eglExts = query_string(EGL_NO_DISPLAY, EGL_EXTENSIONS);
 
   if(eglExts && strstr(eglExts, "EGL_ANDROID_GLES_layers"))