From 69c668c27c26ad7b6157a57788a86eb85c62b456 Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Wed, 30 Jan 2019 20:32:53 +0000
Subject: [PATCH] Suppress GL hooks on apple while in CGLFlushDrawable

---
 renderdoc/driver/gl/cgl_hooks.cpp       | 19 ++++++++++++++++++-
 renderdoc/driver/gl/gl_common.h         | 18 +++++++++---------
 renderdoc/driver/gl/gl_dispatch_table.h |  5 ++++-
 renderdoc/driver/gl/gl_hooks.cpp        |  7 ++++++-
 4 files changed, 37 insertions(+), 12 deletions(-)

diff --git a/renderdoc/driver/gl/cgl_hooks.cpp b/renderdoc/driver/gl/cgl_hooks.cpp
index 3026bfe40..55ff14ad3 100644
--- a/renderdoc/driver/gl/cgl_hooks.cpp
+++ b/renderdoc/driver/gl/cgl_hooks.cpp
@@ -37,6 +37,8 @@ public:
   void *handle = RTLD_NEXT;
   WrappedOpenGL driver;
   std::set<CGLContextObj> contexts;
+
+  volatile int32_t suppressed = 0;
 } cglhook;
 
 CGLError GL_EXPORT_NAME(CGLCreateContext)(CGLPixelFormatObj pix, CGLContextObj share,
@@ -100,6 +102,9 @@ CGLError GL_EXPORT_NAME(CGLSetCurrentContext)(CGLContextObj ctx)
 
   CGLError ret = CGL.CGLSetCurrentContext(ctx);
 
+  if(Atomic::CmpExch32(&cglhook.suppressed, 0, 0) != 0)
+    return ret;
+
   if(ret == kCGLNoError)
   {
     SCOPED_LOCK(glLock);
@@ -173,7 +178,16 @@ CGLError GL_EXPORT_NAME(CGLFlushDrawable)(CGLContextObj ctx)
     cglhook.driver.SwapBuffers((void *)(uintptr_t)window);
   }
 
-  return CGL.CGLFlushDrawable(ctx);
+  CGLError ret;
+  {
+    DisableGLHooks();
+    Atomic::Inc32(&cglhook.suppressed);
+    ret = CGL.CGLFlushDrawable(ctx);
+    Atomic::Dec32(&cglhook.suppressed);
+    EnableGLHooks();
+  }
+
+  return ret;
 }
 
 DECL_HOOK_EXPORT(CGLCreateContext);
@@ -188,6 +202,9 @@ static void CGLHooked(void *handle)
   // pointers
   cglhook.handle = handle;
 
+  // enable hooks immediately, we'll suppress them when calling into CGL
+  EnableGLHooks();
+
   // as a hook callback this is only called while capturing
   RDCASSERT(!RenderDoc::Inst().IsReplayApp());
 
diff --git a/renderdoc/driver/gl/gl_common.h b/renderdoc/driver/gl/gl_common.h
index 459b51902..ec0f36cf0 100644
--- a/renderdoc/driver/gl/gl_common.h
+++ b/renderdoc/driver/gl/gl_common.h
@@ -341,15 +341,15 @@ class IReplayDriver;
 typedef void *HANDLE;
 typedef long BOOL;
 
-typedef BOOL(APIENTRYP *PFNWGLDXSETRESOURCESHAREHANDLENVPROC)(void *dxObject, HANDLE shareHandle);
-typedef HANDLE(APIENTRYP *PFNWGLDXOPENDEVICENVPROC)(void *dxDevice);
-typedef BOOL(APIENTRYP *PFNWGLDXCLOSEDEVICENVPROC)(HANDLE hDevice);
-typedef HANDLE(APIENTRYP *PFNWGLDXREGISTEROBJECTNVPROC)(HANDLE hDevice, void *dxObject, GLuint name,
-                                                        GLenum type, GLenum access);
-typedef BOOL(APIENTRYP *PFNWGLDXUNREGISTEROBJECTNVPROC)(HANDLE hDevice, HANDLE hObject);
-typedef BOOL(APIENTRYP *PFNWGLDXOBJECTACCESSNVPROC)(HANDLE hObject, GLenum access);
-typedef BOOL(APIENTRYP *PFNWGLDXLOCKOBJECTSNVPROC)(HANDLE hDevice, GLint count, HANDLE *hObjects);
-typedef BOOL(APIENTRYP *PFNWGLDXUNLOCKOBJECTSNVPROC)(HANDLE hDevice, GLint count, HANDLE *hObjects);
+typedef BOOL(APIENTRY *PFNWGLDXSETRESOURCESHAREHANDLENVPROC)(void *dxObject, HANDLE shareHandle);
+typedef HANDLE(APIENTRY *PFNWGLDXOPENDEVICENVPROC)(void *dxDevice);
+typedef BOOL(APIENTRY *PFNWGLDXCLOSEDEVICENVPROC)(HANDLE hDevice);
+typedef HANDLE(APIENTRY *PFNWGLDXREGISTEROBJECTNVPROC)(HANDLE hDevice, void *dxObject, GLuint name,
+                                                       GLenum type, GLenum access);
+typedef BOOL(APIENTRY *PFNWGLDXUNREGISTEROBJECTNVPROC)(HANDLE hDevice, HANDLE hObject);
+typedef BOOL(APIENTRY *PFNWGLDXOBJECTACCESSNVPROC)(HANDLE hObject, GLenum access);
+typedef BOOL(APIENTRY *PFNWGLDXLOCKOBJECTSNVPROC)(HANDLE hDevice, GLint count, HANDLE *hObjects);
+typedef BOOL(APIENTRY *PFNWGLDXUNLOCKOBJECTSNVPROC)(HANDLE hDevice, GLint count, HANDLE *hObjects);
 #endif
 
 #include "api/replay/renderdoc_replay.h"
diff --git a/renderdoc/driver/gl/gl_dispatch_table.h b/renderdoc/driver/gl/gl_dispatch_table.h
index 6bfb81b9f..0155dff02 100644
--- a/renderdoc/driver/gl/gl_dispatch_table.h
+++ b/renderdoc/driver/gl/gl_dispatch_table.h
@@ -981,8 +981,11 @@ void SetDriverForHooks(WrappedOpenGL *driver);
 // we see context creation, to prevent crashes trying to handle function calls having seen no
 // intialisation. This can have false positives if the program creates a context late, but it's the
 // best we can do.
-#if ENABLED(RDOC_WIN32)
+// On apple we suppress hooks while entering any CGL function so we don't record internal work that
+// can mess up the replay
+#if ENABLED(RDOC_WIN32) || ENABLED(RDOC_APPLE)
 void EnableGLHooks();
+void DisableGLHooks();
 #else
 #define EnableGLHooks() (void)0
 #endif
diff --git a/renderdoc/driver/gl/gl_hooks.cpp b/renderdoc/driver/gl/gl_hooks.cpp
index 2d14b3320..bc70f5daf 100644
--- a/renderdoc/driver/gl/gl_hooks.cpp
+++ b/renderdoc/driver/gl/gl_hooks.cpp
@@ -105,12 +105,17 @@ void SetDriverForHooks(WrappedOpenGL *driver)
   glhook.driver = driver;
 }
 
-#if ENABLED(RDOC_WIN32)
+#if ENABLED(RDOC_WIN32) || ENABLED(RDOC_APPLE)
 void EnableGLHooks()
 {
   glhook.enabled = true;
 }
 
+void DisableGLHooks()
+{
+  glhook.enabled = false;
+}
+
 // if we were injected and aren't ready to capture, skip out and call the real function
 #define UNINIT_CALL(function, ...)                                                      \
   if(!glhook.enabled)                                                                   \