Use library location not binary location to locate relative paths

* This means that things will work successfully even if the 'executable' is
  actually e.g. python3 in a system directory and nowhere related to where the
  renderdoc library is.

renderdoc/android/android.cpp

@@ -330,9 +330,9 @@ bool InstallRenderDocServer(const std::string &deviceID)
   }
 
   // Check known paths for RenderDoc server
-  std::string exePath;
-  FileIO::GetExecutableFilename(exePath);
-  std::string exeDir = dirname(FileIO::GetFullPathname(exePath));
+  std::string libPath;
+  FileIO::GetLibraryFilename(libPath);
+  std::string libDir = dirname(FileIO::GetFullPathname(libPath));
 
   std::vector<std::string> paths;
 
@@ -341,7 +341,7 @@ bool InstallRenderDocServer(const std::string &deviceID)
   RDCLOG("Custom APK path: %s", customPath.c_str());
 
   if(FileIO::IsRelativePath(customPath))
-    customPath = exeDir + "/" + customPath;
+    customPath = libDir + "/" + customPath;
 
   if(!endswith(customPath, "/"))
     customPath += "/";
@@ -352,12 +352,12 @@ bool InstallRenderDocServer(const std::string &deviceID)
   std::string suff = GetRenderDocPackageForABI(abis[0], '-');
   suff.erase(0, strlen(RENDERDOC_ANDROID_PACKAGE_BASE));
 
-  paths.push_back(exeDir + "/plugins/android/");                                 // Windows install
-  paths.push_back(exeDir + "/../share/renderdoc/plugins/android/");              // Linux install
-  paths.push_back(exeDir + "/../../build-android/bin/");                         // Local build
-  paths.push_back(exeDir + "/../../build-android" + suff + "/bin/");             // Local ABI build
-  paths.push_back(exeDir + "/../../../../../build-android/bin/");                // macOS build
-  paths.push_back(exeDir + "/../../../../../build-android" + suff + "/bin/");    // macOS ABI build
+  paths.push_back(libDir + "/plugins/android/");                                 // Windows install
+  paths.push_back(libDir + "/../share/renderdoc/plugins/android/");              // Linux install
+  paths.push_back(libDir + "/../../build-android/bin/");                         // Local build
+  paths.push_back(libDir + "/../../build-android" + suff + "/bin/");             // Local ABI build
+  paths.push_back(libDir + "/../../../../../build-android/bin/");                // macOS build
+  paths.push_back(libDir + "/../../../../../build-android" + suff + "/bin/");    // macOS ABI build
 
   // use the first ABI for searching
   std::string apk = GetRenderDocPackageForABI(abis[0]);

renderdoc/android/android_tools.cpp

@@ -249,11 +249,11 @@ std::string getToolPath(ToolDir subdir, const std::string &toolname, bool checkE
 
   // finally try to locate it in our own distributed android subfolder
   {
-    std::string exepath;
-    FileIO::GetExecutableFilename(exepath);
-    std::string exedir = dirname(FileIO::GetFullPathname(exepath));
+    std::string libpath;
+    FileIO::GetLibraryFilename(libpath);
+    std::string libdir = dirname(FileIO::GetFullPathname(libpath));
 
-    toolpath = exedir + "/plugins/android/" + toolname;
+    toolpath = libdir + "/plugins/android/" + toolname;
     if(toolExists(toolpath))
     {
       if(toolname == "adb")

renderdoc/core/plugins.cpp

@@ -30,9 +30,9 @@ std::string LocatePluginFile(const std::string &path, const std::string &fileNam
 {
   std::string ret;
 
-  std::string exepath;
-  FileIO::GetExecutableFilename(exepath);
-  exepath = dirname(exepath);
+  std::string libpath;
+  FileIO::GetLibraryFilename(libpath);
+  libpath = dirname(libpath);
 
   std::vector<std::string> paths;
 
@@ -40,32 +40,32 @@ std::string LocatePluginFile(const std::string &path, const std::string &fileNam
   string customPath(RENDERDOC_PLUGINS_PATH);
 
   if(FileIO::IsRelativePath(customPath))
-    customPath = exepath + "/" + customPath;
+    customPath = libpath + "/" + customPath;
 
   paths.push_back(customPath);
 #endif
 
   // windows installation
-  paths.push_back(exepath + "/plugins");
+  paths.push_back(libpath + "/plugins");
   // linux installation
-  paths.push_back(exepath + "/../share/renderdoc/plugins");
+  paths.push_back(libpath + "/../share/renderdoc/plugins");
 // also search the appropriate OS-specific location in the root
 #if ENABLED(RDOC_WIN32) && ENABLED(RDOC_X64)
-  paths.push_back(exepath + "/../../plugins-win64");
+  paths.push_back(libpath + "/../../plugins-win64");
 #endif
 
 #if ENABLED(RDOC_WIN32) && DISABLED(RDOC_X64)
-  paths.push_back(exepath + "/../../plugins-win32");
+  paths.push_back(libpath + "/../../plugins-win32");
 #endif
 
 #if ENABLED(RDOC_LINUX)
-  paths.push_back(exepath + "/../../plugins-linux64");
+  paths.push_back(libpath + "/../../plugins-linux64");
 #endif
 
   // there is no standard path for local builds as we don't provide these plugins in the repository
   // directly. As a courtesy we search the root of the build, from the executable. The user can
   // always put the plugins folder relative to the exe where it would be in an installation too.
-  paths.push_back(exepath + "/../../plugins");
+  paths.push_back(libpath + "/../../plugins");
 
   // in future maybe we want to search a user-specific plugins folder? Like ~/.renderdoc/ on linux
   // or %APPDATA%/renderdoc on windows?

renderdoc/driver/vulkan/vk_apple.cpp

@@ -89,17 +89,6 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h
 
 static const char *VulkanLibraryName = "libvulkan.1.dylib";
 
-string GetThisLibPath()
-{
-  Dl_info info;
-  if(dladdr(&VulkanLibraryName, &info))
-  {
-    RDCDEBUG("GetThisLibPath = '%s'", info.dli_fname);
-    return info.dli_fname;
-  }
-  return "";
-}
-
 void *LoadVulkanLibrary()
 {
   // first try to load the module globally. If so we assume the user has a global (or at least

renderdoc/driver/vulkan/vk_linux.cpp

@@ -239,92 +239,3 @@ void *LoadVulkanLibrary()
 {
   return Process::LoadModule("libvulkan.so.1");
 }
-
-string GetThisLibPath()
-{
-  // this is a hack, but the only reliable way to find the absolute path to the library.
-  // dladdr would be fine but it returns the wrong result for symbols in the library
-
-  string librenderdoc_path;
-
-  FILE *f = fopen("/proc/self/maps", "r");
-
-  if(f)
-  {
-    // read the whole thing in one go. There's no need to try and be tight with
-    // this allocation, so just make sure we can read everything.
-    char *map_string = new char[1024 * 1024];
-    memset(map_string, 0, 1024 * 1024);
-
-    fread(map_string, 1, 1024 * 1024, f);
-
-    fclose(f);
-
-    char *c = strstr(map_string, "/librenderdoc.so");
-
-    if(c)
-    {
-      // walk backwards until we hit the start of the line
-      while(c > map_string)
-      {
-        c--;
-
-        if(c[0] == '\n')
-        {
-          c++;
-          break;
-        }
-      }
-
-      // walk forwards across the address range (00400000-0040c000)
-      while(isalnum(c[0]) || c[0] == '-')
-        c++;
-
-      // whitespace
-      while(c[0] == ' ')
-        c++;
-
-      // permissions (r-xp)
-      while(isalpha(c[0]) || c[0] == '-')
-        c++;
-
-      // whitespace
-      while(c[0] == ' ')
-        c++;
-
-      // offset (0000b000)
-      while(isalnum(c[0]) || c[0] == '-')
-        c++;
-
-      // whitespace
-      while(c[0] == ' ')
-        c++;
-
-      // dev
-      while(isalnum(c[0]) || c[0] == ':')
-        c++;
-
-      // whitespace
-      while(c[0] == ' ')
-        c++;
-
-      // inode
-      while(isdigit(c[0]))
-        c++;
-
-      // whitespace
-      while(c[0] == ' ')
-        c++;
-
-      // FINALLY we are at the start of the actual path
-      char *end = strchr(c, '\n');
-
-      if(end)
-        librenderdoc_path = string(c, end - c);
-    }
-
-    delete[] map_string;
-  }
-
-  return librenderdoc_path;
-}

renderdoc/driver/vulkan/vk_posix.cpp

@@ -181,9 +181,6 @@ void WrappedVulkan::AddRequiredExtensions(bool instance, vector<string> &extensi
   }
 }
 
-// defined in vk_linux.cpp or vk_apple.cpp
-string GetThisLibPath();
-
 // embedded data file
 
 extern unsigned char driver_vulkan_renderdoc_json[];
@@ -343,7 +340,8 @@ bool VulkanReplay::CheckVulkanLayer(VulkanLayerFlags &flags, std::vector<std::st
   // check that there's only one layer registered, and it points to the same .so file that
   // we are running with in this instance of renderdoccmd
 
-  string librenderdoc_path = GetThisLibPath();
+  string librenderdoc_path;
+  FileIO::GetLibraryFilename(librenderdoc_path);
 
   if(librenderdoc_path.empty() || !FileExists(librenderdoc_path))
   {
@@ -454,7 +452,8 @@ void VulkanReplay::InstallVulkanLayer(bool systemLevel)
 
   string jsonPath = LayerRegistrationPath(idx);
   string path = GetSOFromJSON(jsonPath);
-  string libPath = GetThisLibPath();
+  string libPath;
+  FileIO::GetLibraryFilename(libPath);
 
   if(path != libPath)
   {

renderdoc/os/os_specific.h

@@ -278,6 +278,7 @@ string GetFullPathname(const string &filename);
 string FindFileInPath(const string &fileName);
 
 void GetExecutableFilename(string &selfName);
+void GetLibraryFilename(string &selfName);
 
 uint64_t GetModifiedTimestamp(const string &filename);
 

renderdoc/os/posix/android/android_stringio.cpp

@@ -92,6 +92,12 @@ void GetExecutableFilename(string &selfName)
 
   selfName = buf;
 }
+
+void GetLibraryFilename(string &selfName)
+{
+  RDCERR("GetLibraryFilename is not defined on Android");
+  GetExecutableFilename(selfName);
+}
 };
 
 namespace StringFormat

renderdoc/os/posix/apple/apple_stringio.cpp

@@ -108,6 +108,22 @@ void GetExecutableFilename(string &selfName)
   if(path[0] != 0)
     selfName = string(path);
 }
+
+int LibraryLocator = 42;
+
+void GetLibraryFilename(string &selfName)
+{
+  Dl_info info;
+  if(dladdr(&LibraryLocator, &info))
+  {
+    selfName = info.dli_fname;
+  }
+  else
+  {
+    RDCERR("dladdr failed to get library path");
+  }
+  selfName = "";
+}
 };
 
 namespace StringFormat

renderdoc/os/posix/linux/linux_stringio.cpp

@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  ******************************************************************************/
 
+#include <dlfcn.h>
 #include <errno.h>
 #include <iconv.h>
 #include <pwd.h>
@@ -282,6 +283,106 @@ void GetExecutableFilename(string &selfName)
 
   selfName = string(path);
 }
+
+int LibraryLocator = 42;
+
+void GetLibraryFilename(string &selfName)
+{
+  // this is a hack, but the only reliable way to find the absolute path to the library.
+  // dladdr would be fine but it returns the wrong result for symbols in the library
+
+  string librenderdoc_path;
+
+  FILE *f = fopen("/proc/self/maps", "r");
+
+  if(f)
+  {
+    // read the whole thing in one go. There's no need to try and be tight with
+    // this allocation, so just make sure we can read everything.
+    char *map_string = new char[1024 * 1024];
+    memset(map_string, 0, 1024 * 1024);
+
+    ::fread(map_string, 1, 1024 * 1024, f);
+
+    ::fclose(f);
+
+    char *c = strstr(map_string, "/librenderdoc.so");
+
+    if(c)
+    {
+      // walk backwards until we hit the start of the line
+      while(c > map_string)
+      {
+        c--;
+
+        if(c[0] == '\n')
+        {
+          c++;
+          break;
+        }
+      }
+
+      // walk forwards across the address range (00400000-0040c000)
+      while(isalnum(c[0]) || c[0] == '-')
+        c++;
+
+      // whitespace
+      while(c[0] == ' ')
+        c++;
+
+      // permissions (r-xp)
+      while(isalpha(c[0]) || c[0] == '-')
+        c++;
+
+      // whitespace
+      while(c[0] == ' ')
+        c++;
+
+      // offset (0000b000)
+      while(isalnum(c[0]) || c[0] == '-')
+        c++;
+
+      // whitespace
+      while(c[0] == ' ')
+        c++;
+
+      // dev
+      while(isalnum(c[0]) || c[0] == ':')
+        c++;
+
+      // whitespace
+      while(c[0] == ' ')
+        c++;
+
+      // inode
+      while(isdigit(c[0]))
+        c++;
+
+      // whitespace
+      while(c[0] == ' ')
+        c++;
+
+      // FINALLY we are at the start of the actual path
+      char *end = strchr(c, '\n');
+
+      if(end)
+        librenderdoc_path = string(c, end - c);
+    }
+
+    delete[] map_string;
+  }
+
+  if(librenderdoc_path.empty())
+  {
+    RDCWARN("Couldn't get librenderdoc.so path from /proc/self/maps, falling back to dladdr");
+
+    Dl_info info;
+    if(dladdr(&LibraryLocator, &info))
+      librenderdoc_path = info.dli_fname;
+  }
+
+  selfName = librenderdoc_path;
+}
 };
 
 namespace StringFormat

renderdoc/os/posix/posix_libentry.cpp

@@ -31,9 +31,6 @@ void dlopen_hook_init();
 // DllMain equivalent
 void library_loaded()
 {
-  string curfile;
-  FileIO::GetExecutableFilename(curfile);
-
   if(LibraryHooks::Detect("renderdoc__replay__marker"))
   {
     RDCDEBUG("Not creating hooks - in replay app");
@@ -66,6 +63,9 @@ void library_loaded()
       RenderDoc::Inst().SetCaptureFileTemplate(capturefile);
     }
 
+    string curfile;
+    FileIO::GetExecutableFilename(curfile);
+
     RDCLOG("Loading into %s", curfile.c_str());
 
     LibraryHooks::RegisterHooks();

renderdoc/os/posix/posix_process.cpp

@@ -711,7 +711,7 @@ ExecuteResult Process::LaunchAndInjectIntoProcess(const char *app, const char *w
   if(capturefile == NULL)
     capturefile = "";
 
-  string binpath, libpath;
+  string binpath, libpath, ownlibpath;
   {
     FileIO::GetExecutableFilename(binpath);
     binpath = dirname(binpath);
@@ -727,6 +727,9 @@ ExecuteResult Process::LaunchAndInjectIntoProcess(const char *app, const char *w
 #endif
   }
 
+  FileIO::GetLibraryFilename(ownlibpath);
+  ownlibpath = dirname(ownlibpath);
+
   std::string libfile = "librenderdoc" LIB_SUFFIX;
 
 // on macOS, the path must be absolute
@@ -740,6 +743,8 @@ ExecuteResult Process::LaunchAndInjectIntoProcess(const char *app, const char *w
       EnvironmentModification(EnvMod::Append, EnvSep::Platform, LIB_PATH_ENV_VAR, binpath.c_str()));
   modifications.push_back(
       EnvironmentModification(EnvMod::Append, EnvSep::Platform, LIB_PATH_ENV_VAR, libpath.c_str()));
+  modifications.push_back(EnvironmentModification(EnvMod::Append, EnvSep::Platform,
+                                                  LIB_PATH_ENV_VAR, ownlibpath.c_str()));
   modifications.push_back(
       EnvironmentModification(EnvMod::Append, EnvSep::Platform, PRELOAD_ENV_VAR, libfile.c_str()));
   modifications.push_back(

renderdoc/os/win32/win32_stringio.cpp

@@ -168,6 +168,14 @@ void GetExecutableFilename(string &selfName)
   selfName = StringFormat::Wide2UTF8(wstring(curFile));
 }
 
+void GetLibraryFilename(string &selfName)
+{
+  wchar_t curFile[512] = {0};
+  GetModuleFileNameW(GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll"), curFile, 511);
+
+  selfName = StringFormat::Wide2UTF8(wstring(curFile));
+}
+
 bool IsRelativePath(const string &path)
 {
   if(path.empty())