Add dlsym hooking for extreme cases on linux. Closes #1551

renderdoc/CMakeLists.txt

@@ -7,6 +7,13 @@ set(RDOC_INCLUDES
     PRIVATE ${RDOC_SOURCE_DIR}/3rdparty)
 set(RDOC_LIBRARIES)
 
+option(ENABLE_DLSYM_HOOKING "Enable dlsym() hooking via glibc internals" OFF)
+
+if(ENABLE_DLSYM_HOOKING)
+    set(RDOC_DEFINITIONS ${RDOC_DEFINITIONS} PRIVATE -DRENDERDOC_HOOK_DLSYM)
+    message(WARNING "Enabling dlsym() hooking - may cause issues, segfaults, crashes, or bad behaviour. Only enable if absolutely required")
+endif()
+
 if(ANDROID)
     list(APPEND RDOC_LIBRARIES
         PRIVATE -lm

renderdoc/core/core.cpp

@@ -373,6 +373,10 @@ void RenderDoc::Initialise()
   RDCLOG("Packaged for %s (%s) - %s", DISTRIBUTION_NAME, DISTRIBUTION_VERSION, DISTRIBUTION_CONTACT);
 #endif
 
+#if defined(RENDERDOC_HOOK_DLSYM)
+  RDCWARN("dlsym() hooking enabled!");
+#endif
+
   Keyboard::Init();
 
   m_FrameTimer.InitTimers();

renderdoc/os/posix/linux/linux_hook.cpp

@@ -43,7 +43,9 @@ void *intercept_dlopen(const char *filename, int flag, void *ret);
 void plthook_lib(void *handle);
 
 typedef void *(*DLOPENPROC)(const char *, int);
+typedef void *(*DLSYMPROC)(void *, const char *);
 DLOPENPROC realdlopen = NULL;
+DLSYMPROC realdlsym = NULL;
 
 static volatile int32_t tlsbusyflag = 0;
 
@@ -75,6 +77,45 @@ __attribute__((visibility("default"))) void *dlopen(const char *filename, int fl
   return ret;
 }
 
+#if defined(RENDERDOC_HOOK_DLSYM)
+
+#pragma message("ALERT: dlsym() hooking enabled! This is unreliable & relies on glibc internals.")
+
+extern "C" {
+
+__attribute__((visibility("default"))) void *dlsym(void *handle, const char *name);
+
+extern void *_dl_sym(void *, const char *, void *);
+
+void bootstrap_dlsym()
+{
+  realdlsym = (DLSYMPROC)_dl_sym(RTLD_NEXT, "dlsym", (void *)&dlsym);
+}
+
+__attribute__((visibility("default"))) void *dlsym(void *handle, const char *name)
+{
+  if(!strcmp(name, "dlsym"))
+    return (void *)&dlsym;
+
+  if(!strcmp(name, "dlopen") && realdlopen)
+    return (void *)&dlopen;
+
+  if(realdlsym == NULL)
+    bootstrap_dlsym();
+
+  if(realdlsym == NULL)
+  {
+    fprintf(stderr, "Couldn't get onwards dlsym in hooked dlsym\n");
+    exit(-1);
+  }
+
+  return realdlsym(handle, name);
+}
+
+};    // extern "C"
+
+#endif
+
 void plthook_lib(void *handle)
 {
   plthook_t *plthook = NULL;

renderdoc/renderdoc.version

@@ -6,6 +6,7 @@
                 egl[A-Z]*;
                 vk_icd*;
                 dlopen;
+                dlsym;
                 _exit;
                 RENDERDOC_*;
                 VK_LAYER_RENDERDOC_*;