Try to prevent android library being unloaded after it was loaded

* Even if the vulkan/GL loader wants to unload us the library should stay
  resident to provide a persistent connection, as if it were preloaded/injected.

renderdoc/driver/vulkan/vk_layer.cpp

@@ -42,6 +42,52 @@
 #define VK_LAYER_EXPORT extern "C" __declspec(dllexport)
 #endif
 
+#if ENABLED(RDOC_ANDROID)
+#include <dlfcn.h>
+
+void KeepLayerAlive()
+{
+  static bool done = false;
+  if(done)
+    return;
+  done = true;
+
+  // on Android 10 the library only gets loaded for layers. If an instance is destroyed the library
+  // would be unloaded. That could cause us to drop target control connections etc.
+  // we create our own instance, which increases the refcount on the layer, then leak it to prevent
+  // the layer being unloaded.
+  RDCLOG("Creating internal instance to bump layer refcount");
+  void *module = dlopen("libvulkan.so.1", RTLD_NOW | RTLD_LOCAL);
+  if(!module)
+    module = dlopen("libvulkan.so", RTLD_NOW | RTLD_LOCAL);
+
+  if(module)
+  {
+    PFN_vkCreateInstance create = (PFN_vkCreateInstance)dlsym(module, "vkCreateInstance");
+    VkApplicationInfo app = {
+        VK_STRUCTURE_TYPE_APPLICATION_INFO, NULL,
+        "RenderDoc forced instance",        VK_MAKE_VERSION(1, 0, 0),
+        "RenderDoc forced instance",        VK_MAKE_VERSION(1, 0, 0),
+        VK_MAKE_VERSION(1, 0, 0),
+    };
+    VkInstanceCreateInfo info = {
+        VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO, NULL, 0, &app, 0, NULL, 0, NULL,
+    };
+    VkInstance forceLiveInstance = VK_NULL_HANDLE;
+    VkResult vkr = create(&info, NULL, &forceLiveInstance);
+    RDCLOG("Created own instance %p: %s", forceLiveInstance, ToStr(vkr).c_str());
+  }
+  else
+  {
+    RDCERR("Couldn't load libvulkan - can't force layer to stay alive");
+  }
+}
+#else
+void KeepLayerAlive()
+{
+}
+#endif
+
 // we don't actually hook any modules here. This is just used so that it's called
 // at the right time in initialisation (after capture options are available) to
 // set environment variables
@@ -186,6 +232,8 @@ VKAPI_ATTR VkResult VKAPI_CALL hooked_vkCreateInstance(const VkInstanceCreateInf
                                                        const VkAllocationCallbacks *pAllocator,
                                                        VkInstance *pInstance)
 {
+  KeepLayerAlive();
+
   WrappedVulkan *core = new WrappedVulkan();
   return core->vkCreateInstance(pCreateInfo, pAllocator, pInstance);
 }

renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp

@@ -723,7 +723,16 @@ VkResult WrappedVulkan::vkCreateInstance(const VkInstanceCreateInfo *pCreateInfo
 
   InitInstanceExtensionTables(m_Instance, record->instDevInfo);
 
-  RenderDoc::Inst().AddDeviceFrameCapturer(LayerDisp(m_Instance), this);
+  // don't register a frame capturer for our internal instance on android
+  if(pCreateInfo->pApplicationInfo && pCreateInfo->pApplicationInfo->pApplicationName &&
+     rdcstr(pCreateInfo->pApplicationInfo->pApplicationName) == "RenderDoc forced instance")
+  {
+    RDCDEBUG("Not registering internal instance as frame capturer");
+  }
+  else
+  {
+    RenderDoc::Inst().AddDeviceFrameCapturer(LayerDisp(m_Instance), this);
+  }
 
   m_DbgReportCallback = VK_NULL_HANDLE;
   m_DbgUtilsCallback = VK_NULL_HANDLE;

renderdoc/os/posix/android/android_hook.cpp

@@ -728,6 +728,27 @@ void LibraryHooks::EndHookRegistration()
     HOOK_DEBUG_PRINT("%s: %p", lib.c_str(), handle);
   }
 
+  // try to prevent the library from being unloaded, increment our dlopen refcount (might not work
+  // on android, but we'll try!)
+  // we use RTLD_NOLOAD to prevent a second copy being loaded if this path doesn't refer to
+  // ourselves or otherwise breaks because of android's terrible library handling.
+  {
+    rdcstr selfLib;
+    FileIO::GetLibraryFilename(selfLib);
+    if(FileIO::exists(selfLib.c_str()))
+    {
+      void *handle = dlopen(selfLib.c_str(), RTLD_NOW | RTLD_NOLOAD | RTLD_LOCAL);
+      if(handle)
+        RDCLOG("Dummy-loaded %s with dlopen to prevent library unload", selfLib.c_str());
+      else
+        RDCLOG("Failed to dummy-loaded %s with dlopen", selfLib.c_str());
+    }
+    else
+    {
+      RDCLOG("Couldn't dummy-load %s because it doesn't exist", selfLib.c_str());
+    }
+  }
+
   if(libs.empty())
   {
     RDCLOG("No library hooks registered, not doing any hooking");

renderdoc/os/posix/android/android_stringio.cpp

@@ -23,6 +23,8 @@
  ******************************************************************************/
 
 #include <android/log.h>
+#include <ctype.h>
+#include <dlfcn.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include "common/common.h"
@@ -96,10 +98,105 @@ void GetExecutableFilename(rdcstr &selfName)
   selfName = filename;
 }
 
+int LibraryLocator = 42;
+
 void GetLibraryFilename(rdcstr &selfName)
 {
-  RDCERR("GetLibraryFilename is not defined on Android");
-  GetExecutableFilename(selfName);
+  // this is a hack, but the only reliable way to find the absolute path to the library.
+  // dladdr would be fine but it returns the wrong result for symbols in the library
+
+  rdcstr librenderdoc_path;
+
+  FILE *f = fopen("/proc/self/maps", "r");
+
+  if(f)
+  {
+    // read the whole thing in one go. There's no need to try and be tight with
+    // this allocation, so just make sure we can read everything.
+    char *map_string = new char[1024 * 1024];
+    memset(map_string, 0, 1024 * 1024);
+
+    ::fread(map_string, 1, 1024 * 1024, f);
+
+    ::fclose(f);
+
+    char *c = strstr(map_string, "/" RENDERDOC_ANDROID_LIBRARY);
+
+    if(c)
+    {
+      // walk backwards until we hit the start of the line
+      while(c > map_string)
+      {
+        c--;
+
+        if(c[0] == '\n')
+        {
+          c++;
+          break;
+        }
+      }
+
+      // walk forwards across the address range (00400000-0040c000)
+      while(isalnum(c[0]) || c[0] == '-')
+        c++;
+
+      // whitespace
+      while(c[0] == ' ')
+        c++;
+
+      // permissions (r-xp)
+      while(isalpha(c[0]) || c[0] == '-')
+        c++;
+
+      // whitespace
+      while(c[0] == ' ')
+        c++;
+
+      // offset (0000b000)
+      while(isalnum(c[0]) || c[0] == '-')
+        c++;
+
+      // whitespace
+      while(c[0] == ' ')
+        c++;
+
+      // dev
+      while(isalnum(c[0]) || c[0] == ':')
+        c++;
+
+      // whitespace
+      while(c[0] == ' ')
+        c++;
+
+      // inode
+      while(isdigit(c[0]))
+        c++;
+
+      // whitespace
+      while(c[0] == ' ')
+        c++;
+
+      // FINALLY we are at the start of the actual path
+      char *end = strchr(c, '\n');
+
+      if(end)
+        librenderdoc_path = rdcstr(c, end - c);
+    }
+
+    delete[] map_string;
+  }
+
+  if(librenderdoc_path.empty())
+  {
+    RDCWARN("Couldn't get " RENDERDOC_ANDROID_LIBRARY
+            " path from /proc/self/maps, falling back to dladdr");
+
+    Dl_info info;
+    if(dladdr(&LibraryLocator, &info))
+      librenderdoc_path = info.dli_fname;
+  }
+
+  selfName = librenderdoc_path;
 }
 };