diff --git a/renderdoc/driver/vulkan/vk_core.h b/renderdoc/driver/vulkan/vk_core.h index 2222854ce..814034d62 100644 --- a/renderdoc/driver/vulkan/vk_core.h +++ b/renderdoc/driver/vulkan/vk_core.h @@ -385,9 +385,6 @@ private: void FinishCapture(); void EndCaptureFrame(VkImage presentImage); - // TODO - replace this with wrapping VkSurfaceKHRs and - //RENDERDOC_WindowHandle GetHandleForSurface(const VkSurfaceDescriptionKHR* surf); - string MakeRenderPassOpString(bool store); void StartFrameCapture(void *dev, void *wnd); @@ -1357,4 +1354,55 @@ public: IMPLEMENT_FUNCTION_SERIALISED(VkResult, vkQueuePresentKHR, VkQueue queue, const VkPresentInfoKHR* pPresentInfo); + + // these functions are non-serialised as they're only used for windowing + // setup during capture, but they must be intercepted so we can unwrap + // properly + void vkDestroySurfaceKHR( + VkInstance instance, + VkSurfaceKHR surface, + const VkAllocationCallbacks* pAllocator); + +#if defined(VK_USE_PLATFORM_WIN32_KHR) + VkResult vkCreateWin32SurfaceKHR( + VkInstance instance, + HINSTANCE hinstance, + HWND hwnd, + const VkAllocationCallbacks* pAllocator, + VkSurfaceKHR* pSurface); + + VkBool32 vkGetPhysicalDeviceWin32PresentationSupportKHR( + VkPhysicalDevice physicalDevice, + uint32_t queueFamilyIndex); +#endif + +#if defined(VK_USE_PLATFORM_XCB_KHR) + VkResult vkCreateXcbSurfaceKHR( + VkInstance instance, + xcb_connection_t* connection, + xcb_window_t window, + const VkAllocationCallbacks* pAllocator, + VkSurfaceKHR* pSurface); + + VkBool32 vkGetPhysicalDeviceXcbPresentationSupportKHR( + VkPhysicalDevice physicalDevice, + uint32_t queueFamilyIndex, + xcb_connection_t* connection, + xcb_visualid_t visual_id); +#endif + +#if defined(VK_USE_PLATFORM_XLIB_KHR) + VkResult vkCreateXlibSurfaceKHR( + VkInstance instance, + Display* dpy, + Window window, + const VkAllocationCallbacks* pAllocator, + VkSurfaceKHR* pSurface); + + VkBool32 vkGetPhysicalDeviceXlibPresentationSupportKHR( + VkPhysicalDevice physicalDevice, + uint32_t queueFamilyIndex, + Display* dpy, + VisualID visualID); +#endif }; diff --git a/renderdoc/driver/vulkan/vk_hookset_defs.h b/renderdoc/driver/vulkan/vk_hookset_defs.h index 41db9703a..8ea1b1886 100644 --- a/renderdoc/driver/vulkan/vk_hookset_defs.h +++ b/renderdoc/driver/vulkan/vk_hookset_defs.h @@ -24,6 +24,57 @@ #pragma once +#if defined(VK_USE_PLATFORM_WIN32_KHR) + +#define HookInit_PlatformSpecific() \ + HookInit(CreateWin32SurfaceKHR); \ + HookInit(GetPhysicalDeviceWin32PresentationSupportKHR); + +#define HookDefine_PlatformSpecific() \ + HookDefine5(VkResult, vkCreateWin32SurfaceKHR, VkInstance, instance, HINSTANCE, hinstance, HWND, hwnd, const VkAllocationCallbacks*, pAllocator, VkSurfaceKHR*, pSurface); \ + HookDefine2(VkBool32, vkGetPhysicalDeviceWin32PresentationSupportKHR, VkPhysicalDevice, physicalDevice, uint32_t, queueFamilyIndex); \ + +#elif defined(VK_USE_PLATFORM_XCB_KHR) || defined(VK_USE_PLATFORM_XLIB_KHR) + +#if defined(VK_USE_PLATFORM_XCB_KHR) + +#define HookInit_PlatformSpecific_Xcb() \ + HookInit(CreateXcbSurfaceKHR); \ + HookInit(GetPhysicalDeviceXcbPresentationSupportKHR); + +#define HookInit_PlatformSpecific_Xcb() \ + HookDefine5(VkResult, vkCreateXcbSurfaceKHR, VkInstance, instance, xcb_connection_t*, connection, xcb_window_t, window, const VkAllocationCallbacks*, pAllocator, VkSurfaceKHR*, pSurface); \ + HookDefine4(VkBool32, vkGetPhysicalDeviceXcbPresentationSupportKHR, VkPhysicalDevice, physicalDevice, uint32_t, queueFamilyIndex, xcb_connection_t*, connection, xcb_visualid_t, visual_id); \ + +#else + +#define HookInit_PlatformSpecific_Xcb() +#define HookDefine_PlatformSpecific_Xcb() + +#endif + +#if defined(VK_USE_PLATFORM_XLIB_KHR) + +#define HookInit_PlatformSpecific_Xlib() \ + HookInit(CreateXlibSurfaceKHR); \ + HookInit(GetPhysicalDeviceXlibPresentationSupportKHR); + +#define HookInit_PlatformSpecific_Xlib() \ + HookDefine5(VkResult, vkCreateXlibSurfaceKHR, VkInstance, instance, Display*, dpy, Window, window, const VkAllocationCallbacks*, pAllocator, VkSurfaceKHR*, pSurface); \ + HookDefine4(VkBool32, vkGetPhysicalDeviceXlibPresentationSupportKHR, VkPhysicalDevice, physicalDevice, uint32_t, queueFamilyIndex, Display*, dpy, VisualID, visualID); \ + +#else + +#define HookInit_PlatformSpecific_Xlib() +#define HookDefine_PlatformSpecific_Xlib() + +#endif + +#define HookInit_PlatformSpecific() HookInit_PlatformSpecific_Xcb() HookInit_PlatformSpecific_Xlib() +#define HookDefine_PlatformSpecific() HookDefine_PlatformSpecific_Xcb() HookDefine_PlatformSpecific_Xlib() + +#endif + #define HookInitVulkanInstance() \ HookInit(CreateInstance); \ HookInit(DestroyInstance); \ @@ -37,10 +88,12 @@ HookInit(GetPhysicalDeviceMemoryProperties); \ HookInit(DbgCreateMsgCallback); \ HookInit(DbgDestroyMsgCallback); \ + HookInit(DestroySurfaceKHR); \ HookInit(GetPhysicalDeviceSurfaceSupportKHR); \ HookInit(GetPhysicalDeviceSurfaceCapabilitiesKHR); \ HookInit(GetPhysicalDeviceSurfaceFormatsKHR); \ HookInit(GetPhysicalDeviceSurfacePresentModesKHR); \ + HookInit_PlatformSpecific() #define HookInitVulkanDevice() \ HookInit(CreateDevice); \ @@ -310,4 +363,6 @@ HookDefine3(void, vkDestroySwapchainKHR, VkDevice, device, VkSwapchainKHR, swapchain, const VkAllocationCallbacks*, pAllocator); \ HookDefine4(VkResult, vkGetSwapchainImagesKHR, VkDevice, device, VkSwapchainKHR, swapchain, uint32_t*, pCount, VkImage*, pSwapchainImages); \ HookDefine6(VkResult, vkAcquireNextImageKHR, VkDevice, device, VkSwapchainKHR, swapchain, uint64_t, timeout, VkSemaphore, semaphore, VkFence, fence, uint32_t*, pImageIndex); \ - HookDefine2(VkResult, vkQueuePresentKHR, VkQueue, queue, VkPresentInfoKHR*, pPresentInfo); + HookDefine2(VkResult, vkQueuePresentKHR, VkQueue, queue, VkPresentInfoKHR*, pPresentInfo); \ + HookDefine3(void, vkDestroySurfaceKHR, VkInstance, instance, VkSurfaceKHR, surface, const VkAllocationCallbacks*, pAllocator); \ + HookDefine_PlatformSpecific() diff --git a/renderdoc/driver/vulkan/vk_linux.cpp b/renderdoc/driver/vulkan/vk_linux.cpp index e1f4bef1b..d2a4cd40d 100644 --- a/renderdoc/driver/vulkan/vk_linux.cpp +++ b/renderdoc/driver/vulkan/vk_linux.cpp @@ -78,38 +78,88 @@ bool VulkanReplay::IsOutputWindowVisible(uint64_t id) return true; } -/* -RENDERDOC_WindowHandle WrappedVulkan::GetHandleForSurface(const VkSurfaceDescriptionKHR* surf) +#if defined(VK_USE_PLATFORM_XCB_KHR) + +VkBool32 WrappedVulkan::vkGetPhysicalDeviceXcbPresentationSupportKHR( + VkPhysicalDevice physicalDevice, + uint32_t queueFamilyIndex, + xcb_connection_t* connection, + xcb_visualid_t visual_id) { - RDCASSERT(surf); - VkSurfaceDescriptionWindowKHR *winDesc = (VkSurfaceDescriptionWindowKHR *)surf; + return ObjDisp(physicalDevice)->GetPhysicalDeviceXcbPresentationSupportKHR(Unwrap(physicalDevice), queueFamilyIndex, connection, visual_id); +} - RDCASSERT(winDesc->platform == VK_PLATFORM_X11_KHR || - winDesc->platform == VK_PLATFORM_XCB_KHR || - winDesc->platform == VK_PLATFORM_WAYLAND_KHR || - winDesc->platform == VK_PLATFORM_MIR_KHR); +VkResult WrappedVulkan::vkCreateXcbSurfaceKHR( + VkInstance instance, + xcb_connection_t* connection, + xcb_window_t window, + const VkAllocationCallbacks* pAllocator, + VkSurfaceKHR* pSurface) +{ + // should not come in here at all on replay + RDCASSERT(m_State >= WRITING); - if(winDesc->platform == VK_PLATFORM_X11_KHR) + VkResult ret = ObjDisp(instance)->CreateXcbSurfaceKHR(Unwrap(instance), connection, window, pAllocator, pSurface); + + if(ret == VK_SUCCESS) { - VkPlatformHandleX11KHR *handle = (VkPlatformHandleX11KHR *)surf->pPlatformHandle; - - // VKTODOLOW Should support X11 here - //Keyboard::UseConnection(handle->dpy); - - return (Drawable)winDesc->pPlatformWindow; - } - else if(winDesc->platform == VK_PLATFORM_XCB_KHR) - { - VkPlatformHandleXcbKHR *handle = (VkPlatformHandleXcbKHR *)surf->pPlatformHandle; - Keyboard::UseConnection(handle->connection); - - return *(xcb_window_t *)winDesc->pPlatformWindow; + GetResourceManager()->WrapResource(Unwrap(instance), *pSurface); + + WrappedVkSurfaceKHR *wrapped = GetWrapped(*pSurface); + + // since there's no point in allocating a full resource record and storing the window + // handle under there somewhere, we just cast. We won't use the resource record for anything + wrapped->record = (VkResourceRecord *)window; + + Keyboard::UseConnection(connection); } - RDCERR("Unsupported platform %u", (uint32_t)winDesc->platform); + return ret; +} - return NULL; -}*/ +#endif + +#if defined(VK_USE_PLATFORM_XLIB_KHR) + +VkBool32 WrappedVulkan::vkGetPhysicalDeviceXlibPresentationSupportKHR( + VkPhysicalDevice physicalDevice, + uint32_t queueFamilyIndex, + Display* dpy, + VisualID visualID) +{ + return ObjDisp(physicalDevice)->GetPhysicalDeviceXlibPresentationSupportKHR(Unwrap(physicalDevice), queueFamilyIndex, dpy, visualID); +} + +VkResult WrappedVulkan::vkCreateXlibSurfaceKHR( + VkInstance instance, + Display* dpy, + Window window, + const VkAllocationCallbacks* pAllocator, + VkSurfaceKHR* pSurface) +{ + // should not come in here at all on replay + RDCASSERT(m_State >= WRITING); + + VkResult ret = ObjDisp(instance)->CreateXlibSurfaceKHR(Unwrap(instance), dpy, window, pAllocator, pSurface); + + if(ret == VK_SUCCESS) + { + GetResourceManager()->WrapResource(Unwrap(instance), *pSurface); + + WrappedVkSurfaceKHR *wrapped = GetWrapped(*pSurface); + + // since there's no point in allocating a full resource record and storing the window + // handle under there somewhere, we just cast. We won't use the resource record for anything + wrapped->record = (VkResourceRecord *)window; + + // VKTODOLOW Should support Xlib here + //Keyboard::UseConnection(dpy); + } + + return ret; +} + +#endif void *LoadVulkanLibrary() { diff --git a/renderdoc/driver/vulkan/vk_win32.cpp b/renderdoc/driver/vulkan/vk_win32.cpp index 610ab743f..782404c3e 100644 --- a/renderdoc/driver/vulkan/vk_win32.cpp +++ b/renderdoc/driver/vulkan/vk_win32.cpp @@ -64,17 +64,44 @@ bool VulkanReplay::IsOutputWindowVisible(uint64_t id) return (IsWindowVisible(m_OutputWindows[id].wnd) == TRUE); } -/* -RENDERDOC_WindowHandle WrappedVulkan::GetHandleForSurface(const VkSurfaceDescriptionKHR* surf) +#if !defined(VK_USE_PLATFORM_WIN32_KHR) +#error "Win32 KHR platform not defined" +#endif + +VkResult WrappedVulkan::vkCreateWin32SurfaceKHR( + VkInstance instance, + HINSTANCE hinstance, + HWND hwnd, + const VkAllocationCallbacks* pAllocator, + VkSurfaceKHR* pSurface) { - RDCASSERT(surf); - VkSurfaceDescriptionWindowKHR *winDesc = (VkSurfaceDescriptionWindowKHR *)surf; + // should not come in here at all on replay + RDCASSERT(m_State >= WRITING); - RDCASSERT(winDesc->platform == VK_PLATFORM_WIN32_KHR); + VkResult ret = ObjDisp(instance)->CreateWin32SurfaceKHR(Unwrap(instance), hinstance, hwnd, pAllocator, pSurface); - return winDesc->pPlatformWindow; + if(ret == VK_SUCCESS) + { + GetResourceManager()->WrapResource(Unwrap(instance), *pSurface); + + WrappedVkSurfaceKHR *wrapped = GetWrapped(*pSurface); + + // since there's no point in allocating a full resource record and storing the window + // handle under there somewhere, we just cast. We won't use the resource record for anything + wrapped->record = (VkResourceRecord *)hwnd; + + Keyboard::AddInputWindow((void *)hwnd); + } + + return ret; +} + +VkBool32 WrappedVulkan::vkGetPhysicalDeviceWin32PresentationSupportKHR( + VkPhysicalDevice physicalDevice, + uint32_t queueFamilyIndex) +{ + return ObjDisp(physicalDevice)->GetPhysicalDeviceWin32PresentationSupportKHR(Unwrap(physicalDevice), queueFamilyIndex); } -*/ void *LoadVulkanLibrary() { diff --git a/renderdoc/driver/vulkan/wrappers/vk_wsi_funcs.cpp b/renderdoc/driver/vulkan/wrappers/vk_wsi_funcs.cpp index ac432ff71..574b27968 100644 --- a/renderdoc/driver/vulkan/wrappers/vk_wsi_funcs.cpp +++ b/renderdoc/driver/vulkan/wrappers/vk_wsi_funcs.cpp @@ -293,6 +293,7 @@ VkResult WrappedVulkan::vkCreateSwapchainKHR( // make sure we can readback to get the screenshot createInfo.imageUsage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + createInfo.surface = Unwrap(createInfo.surface); VkResult ret = ObjDisp(device)->CreateSwapchainKHR(Unwrap(device), &createInfo, pAllocator, pSwapChain); @@ -319,7 +320,8 @@ VkResult WrappedVulkan::vkCreateSwapchainKHR( record->swapInfo = new SwapchainInfo(); SwapchainInfo &swapInfo = *record->swapInfo; - //swapInfo.wndHandle = GetHandleForSurface(pCreateInfo->pSurfaceDescription); + // sneaky casting of window handle into record + swapInfo.wndHandle = (RENDERDOC_WindowHandle)GetRecord(pCreateInfo->surface); { SCOPED_LOCK(m_SwapLookupLock); @@ -639,4 +641,22 @@ VkResult WrappedVulkan::vkQueuePresentKHR( return vkr; } - \ No newline at end of file + +// creation functions are in vk_.cpp + +void WrappedVulkan::vkDestroySurfaceKHR( + VkInstance instance, + VkSurfaceKHR surface, + const VkAllocationCallbacks* pAllocator) +{ + WrappedVkSurfaceKHR *wrapper = GetWrapped(surface); + + // record pointer has window handle packed in + Keyboard::RemoveInputWindow((void *)wrapper->record); + + // now set record pointer back to NULL so no-one tries to delete it + wrapper->record = NULL; + + GetResourceManager()->ReleaseWrappedResource(surface, true); + ObjDisp(instance)->DestroySurfaceKHR(Unwrap(instance), wrapper->real.As(), pAllocator); +}