Speculative fix - try to avoid crashes if hooked dlopen is called early

* This isn't particularly stable/nice, probably needs a complete rethink

renderdoc/hooks/hooks.h

@@ -81,7 +81,7 @@ class Hook
 // just need this for dlsym
 #include <dlfcn.h>
 
-#define HOOKS_BEGIN()
+#define HOOKS_BEGIN() LinuxHookInit()
 #define HOOKS_END()
 #define HOOKS_REMOVE()
 

renderdoc/os/linux/linux_hook.cpp

@@ -35,6 +35,16 @@
 #include <map>
 #include <string>
 
+// depending on symbol resolution, dlopen could get called really early.
+// until we've initialised, just skip any fancy stuff
+static uint32_t hookInited = 0;
+#define HOOK_MAGIC_NUMBER 0xAAF00F00
+
+void LinuxHookInit()
+{
+	hookInited = HOOK_MAGIC_NUMBER;
+}
+
 // need to lock around use of realdlopen and libraryHooks
 Threading::CriticalSection libLock;
 
@@ -52,6 +62,12 @@ DLOPENPROC realdlopen = NULL;
 __attribute__ ((visibility ("default")))
 void *dlopen(const char *filename, int flag)
 {
+	if(hookInited != HOOK_MAGIC_NUMBER)
+	{
+		DLOPENPROC passthru = (DLOPENPROC)dlsym(RTLD_NEXT, "dlopen");
+		return passthru(filename, flag);
+	}
+
 	SCOPED_LOCK(libLock);
 	if(realdlopen == NULL) realdlopen = (DLOPENPROC)dlsym(RTLD_NEXT, "dlopen");
 

renderdoc/os/linux/linux_hook.h

@@ -26,6 +26,8 @@
 
 typedef void (*dlopenCallback)(void *realLib);
 
+void LinuxHookInit();
+
 // if this name is dlopen'd, the real library will be passed
 // to the callback and librenderdoc.so will be returned to user code
 void LinuxHookLibrary(const char *name, dlopenCallback cb);

renderdoc/os/linux/linux_libentry.cpp

@@ -26,6 +26,8 @@
 #include "hooks/hooks.h"
 #include "os/os_specific.h"
 
+void dlopen_hook_init();
+
 void readCapOpts(const char *str, CaptureOptions *opts)
 {
 	// serialise from string with two chars per byte