Add macOS windowing system to public API

* For now it expects a CALayer since this works around a MoltenVK bug

qrenderdoc/Code/AppleUtils.mm

@@ -0,0 +1,18 @@
+#import <Cocoa/Cocoa.h>
+#include <QuartzCore/CAMetalLayer.h>
+
+// taken from Arseny's comment explaining how to make Qt widgets metal compatible:
+// https://github.com/KhronosGroup/MoltenVK/issues/78#issuecomment-369838674
+extern "C" void *makeNSViewMetalCompatible(void *handle)
+{
+  NSView *view = (NSView *)handle;
+  assert([view isKindOfClass:[NSView class]]);
+
+  if(![view.layer isKindOfClass:[CAMetalLayer class]])
+  {
+    [view setLayer:[CAMetalLayer layer]];
+    [view setWantsLayer:YES];
+  }
+
+  return view.layer;
+}

qrenderdoc/Code/CaptureContext.cpp

@@ -316,6 +316,8 @@ void CaptureContext::LoadCaptureThreaded(const QString &captureFile, const QStri
       m_XCBConnection = QX11Info::connection();
     else
       m_X11Display = QX11Info::display();
+#elif defined(RENDERDOC_PLATFORM_APPLE)
+    m_CurWinSystem = WindowingSystem::MacOS;
 #endif
 
     m_StructuredFile = &r->GetStructuredFile();
@@ -1198,7 +1200,7 @@ int CaptureContext::ResourceNameCacheID()
 }
 
 #if defined(RENDERDOC_PLATFORM_APPLE)
-extern "C" void makeNSViewMetalCompatible(void *handle);
+extern "C" void *makeNSViewMetalCompatible(void *handle);
 #endif
 
 WindowingData CaptureContext::CreateWindowingData(QWidget *window)
@@ -1217,6 +1219,14 @@ WindowingData CaptureContext::CreateWindowingData(QWidget *window)
   else
     return CreateXlibWindowingData(m_X11Display, (Drawable)window->winId());
 
+#elif defined(RENDERDOC_PLATFORM_APPLE)
+
+  void *view = (void *)window->winId();
+
+  void *layer = makeNSViewMetalCompatible(view);
+
+  return CreateMacOSWindowingData(layer);
+
 #elif defined(RENDERDOC_PLATFORM_APPLE)
 
   WindowingData ret = {WindowingSystem::Unknown};

qrenderdoc/qrenderdoc.pro

@@ -132,6 +132,10 @@ win32 {
 	QMAKE_CXXFLAGS_WARN_OFF -= -w
 
 	macx: {
+		SOURCES += Code/AppleUtils.mm
+
+		LIBS += -framework Cocoa -framework QuartzCore
+
 		DEFINES += RENDERDOC_PLATFORM_POSIX RENDERDOC_PLATFORM_APPLE
 		ICON = $$OSX_ICONFILE
 		

qrenderdoc/qrenderdoc_local.vcxproj

@@ -1568,6 +1568,7 @@
     </CustomBuild>
   </ItemGroup>
   <ItemGroup>
+    <None Include="Code\AppleUtils.mm" />
     <None Include="Code\pyrenderdoc\container_handling.i" />
     <None Include="Code\pyrenderdoc\ext_refcounts.i" />
     <None Include="Code\pyrenderdoc\pyconversion.h" />

qrenderdoc/qrenderdoc_local.vcxproj.filters

@@ -1087,6 +1087,9 @@
     <None Include="Resources\README.md">
       <Filter>Resources\Files</Filter>
     </None>
+    <None Include="Code\AppleUtils.mm">
+      <Filter>Code</Filter>
+    </None>
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="Resources\qrenderdoc.rc">

renderdoc/api/replay/renderdoc_replay.h

@@ -324,6 +324,10 @@ DOCUMENT(R"(Specifies a windowing system to use for creating an output window.
 .. data:: Android
 
   The windowing data refers to an Android window. See :func:`CreateAndroidWindowingData`.
+
+.. data:: MacOS
+
+  The windowing data refers to a MacOS / OS X CALayer. See :func:`CreateMacOSWindowingData`.
 )");
 enum class WindowingSystem : uint32_t
 {
@@ -332,6 +336,7 @@ enum class WindowingSystem : uint32_t
   Xlib,
   XCB,
   Android,
+  MacOS,
 };
 
 DECLARE_REFLECTION_ENUM(WindowingSystem);
@@ -391,6 +396,11 @@ struct WindowingData
     {
       ANativeWindow *window;
     } android;
+
+    struct
+    {
+      void *layer;
+    } macOS;
   };
 };
 
@@ -466,6 +476,22 @@ inline const WindowingData CreateAndroidWindowingData(ANativeWindow *window)
   return ret;
 }
 
+DOCUMENT(R"(Create a :class:`WindowingData` for an macOS ``CALayer`` handle (as void pointer).
+
+:param CALayer window: The native ``CALayer`` handle for this window.
+:return: A :class:`WindowingData` corresponding to the given window.
+:rtype: WindowingData
+)");
+inline const WindowingData CreateMacOSWindowingData(void *view)
+{
+  WindowingData ret = {};
+
+  ret.system = WindowingSystem::MacOS;
+  ret.macOS.layer = view;
+
+  return ret;
+}
+
 // declare metatype/reflection for ResourceId here as the struct itself is declared before including
 // all relevant headers above
 #if defined(RENDERDOC_QT_COMPAT)

renderdoc/replay/replay_output.cpp

@@ -64,6 +64,11 @@ static uint64_t GetHandle(WindowingData window)
   RDCASSERT(window.system == WindowingSystem::Android);
   return (uint64_t)window.android.window;    // ANativeWindow *
 
+#elif ENABLED(RDOC_APPLE)
+
+  RDCASSERT(window.system == WindowingSystem::MacOS);
+  return (uint64_t)window.macOS.layer;    // CALayer *
+
 #else
   RDCFATAL("No windowing data defined for this platform! Must be implemented for replay outputs");
 #endif