Move dlopen to be hooked centrally on linux, to be multi-API friendly

renderdoc/Makefile

@@ -38,6 +38,7 @@ os/os_specific.o \
 3rdparty/tinyexr/tinyexr.o \
 os/linux/linux_libentry.o \
 os/linux/linux_callstack.o \
+os/linux/linux_hook.o \
 os/linux/linux_network.o \
 os/linux/linux_process.o \
 os/linux/linux_stringio.o \

renderdoc/driver/gl/gl_hooks_linux.cpp

@@ -276,11 +276,20 @@ class OpenGLHook : LibraryHook
 			delete m_GLDriver;
 		}
 
+		static void libHooked(void *realLib)
+		{
+			libGLdlsymHandle = realLib;
+			OpenGLHook::glhooks.CreateHooks(NULL);
+		}
+
 		bool CreateHooks(const char *libName)
 		{
 			if(!m_EnabledHooks)
 				return false;
 
+			if(libName)
+				LinuxHookLibrary("libGL.so", &libHooked);
+
 			bool success = SetupHooks(GL);
 
 			if(!success) return false;
@@ -816,27 +825,6 @@ __GLXextFuncPtr glXGetProcAddressARB(const GLubyte *f)
 	return glXGetProcAddress(f);
 }
 
-typedef void* (*DLOPENPROC)(const char*,int);
-DLOPENPROC realdlopen = NULL;
-
-__attribute__ ((visibility ("default")))
-void *dlopen(const char *filename, int flag)
-{
-	if(realdlopen == NULL) realdlopen = (DLOPENPROC)dlsym(RTLD_NEXT, "dlopen");
-
-	void *ret = realdlopen(filename, flag);
-
-	if(filename && ret && strstr(filename, "libGL.so"))
-	{
-		RDCDEBUG("Redirecting dlopen to ourselves");
-		libGLdlsymHandle = ret;
-		OpenGLHook::glhooks.CreateHooks("libGL.so");
-		ret = realdlopen("librenderdoc.so", flag);
-	}
-
-	return ret;
-}
-
 bool OpenGLHook::PopulateHooks()
 {
 	bool success = true;

renderdoc/hooks/hooks.h

@@ -76,6 +76,8 @@ class Hook
 
 #elif defined(LINUX)
 
+#include "os/linux/linux_hook.h"
+
 // just need this for dlsym
 #include <dlfcn.h>
 

renderdoc/os/linux/linux_hook.cpp

@@ -0,0 +1,77 @@
+/******************************************************************************
+ * The MIT License (MIT)
+ * 
+ * Copyright (c) 2015 Baldur Karlsson
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ ******************************************************************************/
+
+#include "os/os_specific.h"
+
+#include "linux_hook.h"
+
+#include "serialise/string_utils.h"
+#include "common/threading.h"
+
+#include <string.h>
+#include <dlfcn.h>
+
+#include <map>
+#include <string>
+
+// need to lock around use of realdlopen and libraryHooks
+Threading::CriticalSection libLock;
+
+static std::map<std::string, dlopenCallback> libraryHooks;
+
+void LinuxHookLibrary(const char *name, dlopenCallback cb)
+{
+	SCOPED_LOCK(libLock);
+	libraryHooks[name] = cb;
+}
+
+typedef void* (*DLOPENPROC)(const char*,int);
+DLOPENPROC realdlopen = NULL;
+
+__attribute__ ((visibility ("default")))
+void *dlopen(const char *filename, int flag)
+{
+	SCOPED_LOCK(libLock);
+	if(realdlopen == NULL) realdlopen = (DLOPENPROC)dlsym(RTLD_NEXT, "dlopen");
+
+	void *ret = realdlopen(filename, flag);
+
+	if(filename && ret)
+	{
+		for(auto it = libraryHooks.begin(); it != libraryHooks.end(); ++it)
+		{
+			if(strstr(filename, it->first.c_str()))
+			{
+				RDCDEBUG("Redirecting dlopen to ourselves for %s", filename);
+
+				it->second(ret);
+
+				ret = realdlopen("librenderdoc.so", flag);
+			}
+		}
+	}
+
+	return ret;
+}
+

renderdoc/os/linux/linux_hook.h

@@ -0,0 +1,31 @@
+/******************************************************************************
+ * The MIT License (MIT)
+ * 
+ * Copyright (c) 2015 Baldur Karlsson
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ ******************************************************************************/
+
+#pragma once
+
+typedef void (*dlopenCallback)(void *realLib);
+
+// if this name is dlopen'd, the real library will be passed
+// to the callback and librenderdoc.so will be returned to user code
+void LinuxHookLibrary(const char *name, dlopenCallback cb);

renderdoc/renderdoc.vcxproj

@@ -291,6 +291,9 @@
     <ClCompile Include="os\linux\linux_callstack.cpp">
       <ExcludedFromBuild>true</ExcludedFromBuild>
     </ClCompile>
+    <ClCompile Include="os\linux\linux_hook.cpp">
+      <ExcludedFromBuild>true</ExcludedFromBuild>
+    </ClCompile>
     <ClCompile Include="os\linux\linux_libentry.cpp">
       <ExcludedFromBuild>true</ExcludedFromBuild>
     </ClCompile>
@@ -383,4 +386,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

renderdoc/renderdoc.vcxproj.filters

@@ -329,6 +329,9 @@
     <ClCompile Include="os\linux\linux_network.cpp">
       <Filter>OS\Linux</Filter>
     </ClCompile>
+    <ClCompile Include="os\linux\linux_hook.cpp">
+      <Filter>OS\Linux</Filter>
+    </ClCompile>
     <ClCompile Include="os\linux\linux_process.cpp">
       <Filter>OS\Linux</Filter>
     </ClCompile>
@@ -424,4 +427,4 @@
       <Filter>Resources</Filter>
     </ResourceCompile>
   </ItemGroup>
-</Project>
+</Project>