From da5cefbac217cb8a856e863a0e3cf888edadb4fb Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Thu, 11 Jan 2018 12:09:29 +0000
Subject: [PATCH] Add workaround for when vulkan ICD is libGL.so and we repoint
 it to us

* Some distribution packages (Ubuntu at least I think) point the nvidia
  vulkan ICD registration at libGL.so. Since for libGL hooking we have
  to redirect dlopen("libGL.so") to ourselves, this means the vulkan
  loader loads librenderdoc instead of libGL. If we don't export the VK
  ICD entry points, it will fail to load that ICD.
* This solution isn't perfect - we export all three main ICD entry
  points. If the real ICD doesn't export one, we will fail and cause
  problems, since the loader won't fall back to older paths unless the
  symbol is completely missing. For now though this is fine since the
  nvidia ICD exports all three.
* The only way to solve that properly would be to intercept all calls to
  dlsym and redirect these functions by name to go to the real handle
  instead of ourselves - vastly overcomplicated for this edge case.
---
 renderdoc/driver/gl/gl_hooks_linux.cpp | 57 ++++++++++++++++++++++++++
 renderdoc/renderdoc.version            |  1 +
 2 files changed, 58 insertions(+)

diff --git a/renderdoc/driver/gl/gl_hooks_linux.cpp b/renderdoc/driver/gl/gl_hooks_linux.cpp
index aed62384e..ab202b50d 100644
--- a/renderdoc/driver/gl/gl_hooks_linux.cpp
+++ b/renderdoc/driver/gl/gl_hooks_linux.cpp
@@ -799,6 +799,63 @@ __attribute__((visibility("default"))) void glXDestroyWindow(Display *dpy, GLXWi
   return glhooks.glXDestroyWindow_real(dpy, window);
 }
 
+// because we intercept all dlopen calls to "libGL.so*" to ourselves, we can interfere with some
+// vulkan ICDs. For some reason they point the vulkan ICD to that file and so the vulkan loader
+// tries to get the bootstrap entry points from us. I think this is a distribution thing and is not
+// true in the official nvidia package, I'm not sure.
+// Unfortunately there's no perfect way to fix this, since if we declare a function the ICD doesn't
+// support we're screwed. We just have to hope the ICD exports all these functions that we can
+// forward on to.
+
+// declare minimal typedefs to get by
+typedef void *VkInstance;
+enum VkResult
+{
+  VK_ERROR_INCOMPATIBLE_DRIVER = -9
+};
+struct VkNegotiateLayerInterface;
+
+typedef void (*PFN_vkVoidFunction)(void);
+typedef PFN_vkVoidFunction (*PFN_vkGetInstanceProcAddr)(VkInstance instance, const char *pName);
+typedef PFN_vkVoidFunction (*PFN_GetPhysicalDeviceProcAddr)(VkInstance instance, const char *pName);
+typedef VkResult (*PFN_vkNegotiateLoaderLayerInterfaceVersion)(VkNegotiateLayerInterface *pVersionStruct);
+
+__attribute__((visibility("default"))) PFN_vkVoidFunction vk_icdGetInstanceProcAddr(
+    VkInstance instance, const char *pName)
+{
+  PFN_vkGetInstanceProcAddr real =
+      (PFN_vkGetInstanceProcAddr)dlsym(libGLdlsymHandle, "vk_icdGetInstanceProcAddr");
+
+  if(real)
+    return real(instance, pName);
+
+  return NULL;
+}
+
+__attribute__((visibility("default"))) PFN_vkVoidFunction vk_icdGetPhysicalDeviceProcAddr(
+    VkInstance instance, const char *pName)
+{
+  PFN_GetPhysicalDeviceProcAddr real =
+      (PFN_GetPhysicalDeviceProcAddr)dlsym(libGLdlsymHandle, "vk_icdGetPhysicalDeviceProcAddr");
+
+  if(real)
+    return real(instance, pName);
+
+  return NULL;
+}
+
+__attribute__((visibility("default"))) VkResult vk_icdNegotiateLoaderLayerInterfaceVersion(
+    VkNegotiateLayerInterface *pVersionStruct)
+{
+  PFN_vkNegotiateLoaderLayerInterfaceVersion real = (PFN_vkNegotiateLoaderLayerInterfaceVersion)dlsym(
+      libGLdlsymHandle, "vk_icdNegotiateLoaderLayerInterfaceVersion");
+
+  if(real)
+    return real(pVersionStruct);
+
+  return VK_ERROR_INCOMPATIBLE_DRIVER;
+}
+
 };    // extern "C"
 
 bool OpenGLHook::PopulateHooks()
diff --git a/renderdoc/renderdoc.version b/renderdoc/renderdoc.version
index a7396209d..990d2e790 100644
--- a/renderdoc/renderdoc.version
+++ b/renderdoc/renderdoc.version
@@ -4,6 +4,7 @@
                 _fini;
                 gl[A-Z]*;
                 egl[A-Z]*;
+                vk_icd*;
                 dlopen;
                 _exit;
                 RENDERDOC_*;