From 63ff47499db232be7523f25c962fc71d1a4f9a7a Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Tue, 24 Jul 2018 11:28:34 +0100
Subject: [PATCH] Fix fake vulkan hooks to use specific library if available.
 Refs #1040

* RTLD_NEXT won't work if the library is loaded via RTLD_LOCAL (which the vulkan
  loader does). So instead we should keep the actual handle via a registered
  library hook and use that.
---
 renderdoc/driver/gl/glx_fake_vk_hooks.cpp | 41 ++++++++++++++++++++---
 1 file changed, 37 insertions(+), 4 deletions(-)

diff --git a/renderdoc/driver/gl/glx_fake_vk_hooks.cpp b/renderdoc/driver/gl/glx_fake_vk_hooks.cpp
index c0876a049..036d79b35 100644
--- a/renderdoc/driver/gl/glx_fake_vk_hooks.cpp
+++ b/renderdoc/driver/gl/glx_fake_vk_hooks.cpp
@@ -24,6 +24,23 @@
 
 #include <dlfcn.h>
 #include <stddef.h>
+#include "common/common.h"
+#include "hooks/hooks.h"
+
+class FakeVkHook : LibraryHook
+{
+public:
+  void RegisterHooks()
+  {
+    LibraryHooks::RegisterLibraryHook("libGL.so", &FakeVkHooked);
+    LibraryHooks::RegisterLibraryHook("libGL.so.1", &FakeVkHooked);
+  }
+
+  static void FakeVkHooked(void *handle) { searchHandle = handle; }
+  static void *searchHandle;
+} fakevkhook;
+
+void *FakeVkHook::searchHandle = RTLD_NEXT;
 
 extern "C" {
 
@@ -53,23 +70,33 @@ __attribute__((visibility("default"))) PFN_vkVoidFunction vk_icdGetInstanceProcA
     VkInstance instance, const char *pName)
 {
   PFN_vkGetInstanceProcAddr real =
-      (PFN_vkGetInstanceProcAddr)dlsym(RTLD_NEXT, "vk_icdGetInstanceProcAddr");
+      (PFN_vkGetInstanceProcAddr)dlsym(fakevkhook.searchHandle, "vk_icdGetInstanceProcAddr");
+
+  if(!real)
+    real = (PFN_vkGetInstanceProcAddr)dlsym(RTLD_NEXT, "vk_icdGetInstanceProcAddr");
 
   if(real)
     return real(instance, pName);
 
+  RDCERR("Couldn't get real vk_icdGetInstanceProcAddr!");
+
   return NULL;
 }
 
 __attribute__((visibility("default"))) PFN_vkVoidFunction vk_icdGetPhysicalDeviceProcAddr(
     VkInstance instance, const char *pName)
 {
-  PFN_GetPhysicalDeviceProcAddr real =
-      (PFN_GetPhysicalDeviceProcAddr)dlsym(RTLD_NEXT, "vk_icdGetPhysicalDeviceProcAddr");
+  PFN_GetPhysicalDeviceProcAddr real = (PFN_GetPhysicalDeviceProcAddr)dlsym(
+      fakevkhook.searchHandle, "vk_icdGetPhysicalDeviceProcAddr");
+
+  if(!real)
+    real = (PFN_GetPhysicalDeviceProcAddr)dlsym(RTLD_NEXT, "vk_icdGetPhysicalDeviceProcAddr");
 
   if(real)
     return real(instance, pName);
 
+  RDCERR("Couldn't get real vk_icdGetPhysicalDeviceProcAddr!");
+
   return NULL;
 }
 
@@ -77,11 +104,17 @@ __attribute__((visibility("default"))) VkResult vk_icdNegotiateLoaderLayerInterf
     VkNegotiateLayerInterface *pVersionStruct)
 {
   PFN_vkNegotiateLoaderLayerInterfaceVersion real = (PFN_vkNegotiateLoaderLayerInterfaceVersion)dlsym(
-      RTLD_NEXT, "vk_icdNegotiateLoaderLayerInterfaceVersion");
+      fakevkhook.searchHandle, "vk_icdNegotiateLoaderLayerInterfaceVersion");
+
+  if(!real)
+    real = (PFN_vkNegotiateLoaderLayerInterfaceVersion)dlsym(
+        RTLD_NEXT, "vk_icdNegotiateLoaderLayerInterfaceVersion");
 
   if(real)
     return real(pVersionStruct);
 
+  RDCERR("Couldn't get real vk_icdNegotiateLoaderLayerInterfaceVersion!");
+
   return VK_ERROR_INCOMPATIBLE_DRIVER;
 }