Vulkan Mac High DPI fix GetOutputWindowDimensions

Convert layer bounds to backing co-ordinates
Fixes the TextureViewer cursor co-ordinates being incorrect on high DPI displays
Replaced getMetalLayerWidth and getMetalLayerHeight with single call getMetalLayerSize

renderdoc/driver/vulkan/vk_apple.cpp

@@ -29,8 +29,7 @@
 #include <dlfcn.h>
 
 // helpers defined in vk_apple.mm
-extern "C" int getMetalLayerWidth(void *handle);
-extern "C" int getMetalLayerHeight(void *handle);
+void getMetalLayerSize(void *viewHandle, void *layerHandle, int &width, int &height);
 
 #if defined(VK_USE_PLATFORM_MACOS_MVK)
 
@@ -89,7 +88,8 @@ VkResult WrappedVulkan::vkCreateMetalSurfaceEXT(VkInstance instance,
 void VulkanReplay::OutputWindow::SetWindowHandle(WindowingData window)
 {
   RDCASSERT(window.system == WindowingSystem::MacOS, window.system);
-  wnd = window.macOS.layer;
+  cocoa.view = window.macOS.view;
+  cocoa.layer = window.macOS.layer;
 }
 
 void VulkanReplay::OutputWindow::CreateSurface(WrappedVulkan *driver, VkInstance inst)
@@ -102,7 +102,7 @@ void VulkanReplay::OutputWindow::CreateSurface(WrappedVulkan *driver, VkInstance
     createInfo.sType = VK_STRUCTURE_TYPE_METAL_SURFACE_CREATE_INFO_EXT;
     createInfo.pNext = NULL;
     createInfo.flags = 0;
-    createInfo.pLayer = wnd;
+    createInfo.pLayer = cocoa.layer;
 
     RDCDEBUG("Creating macOS surface with EXT_metal_surface");
 
@@ -120,7 +120,7 @@ void VulkanReplay::OutputWindow::CreateSurface(WrappedVulkan *driver, VkInstance
     createInfo.sType = VK_STRUCTURE_TYPE_MACOS_SURFACE_CREATE_INFO_MVK;
     createInfo.pNext = NULL;
     createInfo.flags = 0;
-    createInfo.pView = wnd;
+    createInfo.pView = cocoa.view;
 
     RDCDEBUG("Creating macOS surface with MVK_macos_surface");
 
@@ -147,8 +147,7 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h
     return;
   }
 
-  w = getMetalLayerWidth(outw.wnd);
-  h = getMetalLayerHeight(outw.wnd);
+  getMetalLayerSize(outw.cocoa.view, outw.cocoa.layer, w, h);
 }
 
 static const rdcstr VulkanLibraryName = "libvulkan.1.dylib"_lit;

renderdoc/driver/vulkan/vk_apple.mm

@@ -1,17 +1,13 @@
 #import <Cocoa/Cocoa.h>
 
-extern "C" int getMetalLayerWidth(void *handle)
+void getMetalLayerSize(void *viewHandle, void* layerHandle, int& width, int& height)
 {
-  CALayer *layer = (CALayer *)handle;
+  NSView *view = (NSView *)viewHandle;
+  assert([view isKindOfClass:[NSView class]]);
+  CALayer *layer = (CALayer *)layerHandle;
   assert([layer isKindOfClass:[CALayer class]]);
 
-  return layer.bounds.size.width;
-}
-
-extern "C" int getMetalLayerHeight(void *handle)
-{
-  CALayer *layer = (CALayer *)handle;
-  assert([layer isKindOfClass:[CALayer class]]);
-
-  return layer.bounds.size.height;
+  CGSize viewScale = [view convertSizeToBacking:layer.bounds.size];
+  width = viewScale.width;
+  height = viewScale.height;
 }

renderdoc/driver/vulkan/vk_replay.h

@@ -108,7 +108,18 @@
   RDCEraseEl(xcb);         \
   RDCEraseEl(wayland);
 
-#elif ENABLED(RDOC_APPLE) || ENABLED(RDOC_GGP)
+#elif ENABLED(RDOC_APPLE)
+
+#define WINDOW_HANDLE_DECL \
+  struct                   \
+  {                        \
+    void *view;            \
+    void *layer;           \
+  } cocoa;
+
+#define WINDOW_HANDLE_INIT RDCEraseEl(cocoa);
+
+#elif ENABLED(RDOC_GGP)
 
 #define WINDOW_HANDLE_DECL void *wnd;
 #define WINDOW_HANDLE_INIT wnd = NULL;