Apple: fix hooking not working on OSX 12

Change hooking to use link time symbol address instead of runtime dlsym
Since OSX12 dlsym(RTLD_NEXT,...) returns the interposed symbol not the real symbol

renderdoc/driver/gl/CMakeLists.txt

@@ -54,6 +54,7 @@ list(APPEND sources gl_hooks.cpp)
 if(APPLE)
     list(APPEND sources
         apple_gl_hook_defs.h
+        apple_gl_hook_defs.cpp
         cgl_dispatch_table.h
         cgl_platform_helpers.mm
         cgl_platform.cpp

renderdoc/driver/gl/apple_gl_hook_defs.cpp

@@ -0,0 +1,41 @@
+/******************************************************************************
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019-2022 Baldur Karlsson
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ ******************************************************************************/
+
+#include "apple_gl_hook_defs.h"
+#include "common/common.h"
+
+#define GL_GLEXT_PROTOTYPES
+
+#include "official/glcorearb.h"
+#include "official/glext.h"
+
+extern void AppleRegisterRealSymbol(const char *functionName, void *address);
+
+void RegisterAppleGLSymbols()
+{
+#define APPLE_FUNC(function) AppleRegisterRealSymbol(STRINGIZE(function), (void *)&::function);
+
+  ForEachAppleSupported();
+#undef APPLE_FUNC
+}

renderdoc/driver/gl/cgl_hooks.cpp

@@ -199,6 +199,9 @@ DECL_HOOK_EXPORT(CGLCreateContext);
 DECL_HOOK_EXPORT(CGLSetCurrentContext);
 DECL_HOOK_EXPORT(CGLFlushDrawable);
 
+extern void RegisterAppleGLSymbols();
+extern void AppleRegisterRealSymbol(const char *functionName, void *address);
+
 static void CGLHooked(void *handle)
 {
   RDCDEBUG("CGL library hooked");
@@ -207,6 +210,7 @@ static void CGLHooked(void *handle)
   // pointers
   cglhook.handle = handle;
 
+  RegisterAppleGLSymbols();
   // enable hooks immediately, we'll suppress them when calling into CGL
   EnableGLHooks();
 
@@ -229,8 +233,9 @@ void CGLHook::RegisterHooks()
   LibraryHooks::RegisterLibraryHook("libGL.dylib", NULL);
 
 // register CGL hooks
-#define CGL_REGISTER(func)            \
-  LibraryHooks::RegisterFunctionHook( \
+#define CGL_REGISTER(func)                                   \
+  AppleRegisterRealSymbol(STRINGIZE(func), (void *)&::func); \
+  LibraryHooks::RegisterFunctionHook(                        \
       "OpenGL", FunctionHook(STRINGIZE(func), (void **)&CGL.func, (void *)&GL_EXPORT_NAME(func)));
   CGL_HOOKED_SYMBOLS(CGL_REGISTER)
 #undef CGL_REGISTER

renderdoc/os/posix/apple/apple_hook.cpp

@@ -39,6 +39,7 @@ static std::map<rdcstr, rdcarray<FunctionLoadCallback>> libraryCallbacks;
 static std::set<rdcstr> libraryHooks;
 static rdcarray<FunctionHook> functionHooks;
 static std::set<void *> libraryHandles;
+static std::map<rdcstr, void *> realSymbols;
 
 void *interposed_dlopen(const char *filename, int flag)
 {
@@ -130,7 +131,16 @@ void LibraryHooks::EndHookRegistration()
   {
     if(hook.orig && *hook.orig == NULL)
     {
-      *hook.orig = dlsym(RTLD_NEXT, hook.function.c_str());
+      // Try to get direct compile time function pointer before using dlsym
+      auto it = realSymbols.find(hook.function);
+      if(it != realSymbols.end())
+      {
+        *hook.orig = it->second;
+      }
+      if(*hook.orig == NULL)
+      {
+        *hook.orig = dlsym(RTLD_NEXT, hook.function.c_str());
+      }
       RDCASSERT(*hook.orig != hook.hook, hook.function);
     }
   }
@@ -173,3 +183,8 @@ ScopedSuppressHooking::ScopedSuppressHooking()
 ScopedSuppressHooking::~ScopedSuppressHooking()
 {
 }
+
+void AppleRegisterRealSymbol(const char *functionName, void *address)
+{
+  realSymbols[functionName] = address;
+}