diff --git a/renderdoc/driver/vulkan/vk_core.cpp b/renderdoc/driver/vulkan/vk_core.cpp index c16cec395..af6814bf4 100644 --- a/renderdoc/driver/vulkan/vk_core.cpp +++ b/renderdoc/driver/vulkan/vk_core.cpp @@ -373,7 +373,7 @@ VkCommandBuffer WrappedVulkan::GetExtQueueCmd(uint32_t queueFamilyIdx) const return VK_NULL_HANDLE; } - VkCommandBuffer buf = m_ExternalQueues[queueFamilyIdx].buffer; + VkCommandBuffer buf = m_ExternalQueues[queueFamilyIdx].ring[0].acquire; ObjDisp(buf)->ResetCommandBuffer(Unwrap(buf), 0); @@ -388,7 +388,7 @@ void WrappedVulkan::SubmitAndFlushExtQueue(uint32_t queueFamilyIdx) const return; } - VkCommandBuffer buf = Unwrap(m_ExternalQueues[queueFamilyIdx].buffer); + VkCommandBuffer buf = Unwrap(m_ExternalQueues[queueFamilyIdx].ring[0].acquire); VkSubmitInfo submitInfo = { VK_STRUCTURE_TYPE_SUBMIT_INFO, diff --git a/renderdoc/driver/vulkan/vk_core.h b/renderdoc/driver/vulkan/vk_core.h index b45bb6fa0..9c0555362 100644 --- a/renderdoc/driver/vulkan/vk_core.h +++ b/renderdoc/driver/vulkan/vk_core.h @@ -450,7 +450,22 @@ private: { VkQueue queue = VK_NULL_HANDLE; VkCommandPool pool = VK_NULL_HANDLE; - VkCommandBuffer buffer = VK_NULL_HANDLE; + + struct + { + VkCommandBuffer acquire = VK_NULL_HANDLE; + VkCommandBuffer release = VK_NULL_HANDLE; + VkSemaphore fromext = VK_NULL_HANDLE; + VkSemaphore toext = VK_NULL_HANDLE; + VkFence fence = VK_NULL_HANDLE; + } ring[4]; + uint32_t ringIndex = 0; + uint32_t GetNextIdx() + { + uint32_t ret = ringIndex; + ringIndex = (ringIndex + 1) % ARRAY_COUNT(ring); + return ret; + } }; rdcarray m_ExternalQueues; @@ -938,9 +953,6 @@ private: int32_t messageCode, const char *pLayerPrefix, const char *pMessage, void *pUserData); void AddFrameTerminator(uint64_t queueMarkerTag); - void SubmitExtQBarriers(const std::map> &extQBarriers); - void SubmitExtQBarriers(uint32_t queueFamilyIndex, - const rdcarray &queueFamilyBarriers); VkResourceRecord *RegisterSurface(WindowingSystem system, void *handle); diff --git a/renderdoc/driver/vulkan/vk_initstate.cpp b/renderdoc/driver/vulkan/vk_initstate.cpp index 0de8bc02f..df7148e41 100644 --- a/renderdoc/driver/vulkan/vk_initstate.cpp +++ b/renderdoc/driver/vulkan/vk_initstate.cpp @@ -1334,42 +1334,6 @@ std::map > GetExtQBarriers( return extQBarriers; } -void WrappedVulkan::SubmitExtQBarriers(uint32_t queueFamilyIndex, - const rdcarray &queueFamilyBarriers) -{ - if(queueFamilyBarriers.empty()) - return; - - if(queueFamilyIndex == ~0U) - queueFamilyIndex = queueFamilyBarriers[0].dstQueueFamilyIndex; - - VkCommandBuffer extQCmd = GetExtQueueCmd(queueFamilyIndex); - - VkCommandBufferBeginInfo beginInfo = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, NULL, - VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; - - VkResult vkr = ObjDisp(extQCmd)->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo); - RDCASSERTEQUAL(vkr, VK_SUCCESS); - - DoPipelineBarrier(extQCmd, queueFamilyBarriers.size(), queueFamilyBarriers.data()); - vkr = ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd)); - RDCASSERTEQUAL(vkr, VK_SUCCESS); - - SubmitAndFlushExtQueue(queueFamilyIndex); -} - -void WrappedVulkan::SubmitExtQBarriers( - const std::map > &extQBarriers) -{ - for(auto extQBarrierIt = extQBarriers.begin(); extQBarrierIt != extQBarriers.end(); ++extQBarrierIt) - { - uint32_t queueFamilyIndex = extQBarrierIt->first; - const rdcarray &queueFamilyBarriers = extQBarrierIt->second; - - SubmitExtQBarriers(queueFamilyIndex, queueFamilyBarriers); - } -} - void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, const VkInitialContents &initial) { VkResourceType type = initial.type; diff --git a/renderdoc/driver/vulkan/vk_resources.h b/renderdoc/driver/vulkan/vk_resources.h index 92ab9cfb4..8b1e6051d 100644 --- a/renderdoc/driver/vulkan/vk_resources.h +++ b/renderdoc/driver/vulkan/vk_resources.h @@ -961,6 +961,9 @@ struct SwapchainInfo VkImageView view; VkFramebuffer fb; + + VkCommandBuffer cmd; + VkSemaphore overlaydone; }; rdcarray images; PresentInfo lastPresent; diff --git a/renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp b/renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp index 4fd4ef596..482cd43d4 100644 --- a/renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp +++ b/renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp @@ -803,6 +803,13 @@ void WrappedVulkan::Shutdown() SubmitSemaphores(); FlushQ(); + // idle the device as well so that external queues are idle. + if(m_Device) + { + VkResult vkr = ObjDisp(m_Device)->DeviceWaitIdle(Unwrap(m_Device)); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + } + // destroy any events we created for waiting on for(size_t i = 0; i < m_PersistentEvents.size(); i++) ObjDisp(GetDev())->DestroyEvent(Unwrap(GetDev()), m_PersistentEvents[i], NULL); @@ -833,9 +840,25 @@ void WrappedVulkan::Shutdown() for(size_t i = 0; i < m_ExternalQueues.size(); i++) { - if(m_ExternalQueues[i].buffer != VK_NULL_HANDLE) + if(m_ExternalQueues[i].pool != VK_NULL_HANDLE) { - GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].buffer); + for(size_t x = 0; x < ARRAY_COUNT(m_ExternalQueues[i].ring); x++) + { + GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].acquire); + GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].release); + + ObjDisp(m_Device)->DestroySemaphore(Unwrap(m_Device), + Unwrap(m_ExternalQueues[i].ring[x].fromext), NULL); + GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].fromext); + + ObjDisp(m_Device)->DestroySemaphore(Unwrap(m_Device), + Unwrap(m_ExternalQueues[i].ring[x].toext), NULL); + GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].toext); + + ObjDisp(m_Device)->DestroyFence(Unwrap(m_Device), Unwrap(m_ExternalQueues[i].ring[x].fence), + NULL); + GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].fence); + } ObjDisp(m_Device)->DestroyCommandPool(Unwrap(m_Device), Unwrap(m_ExternalQueues[i].pool), NULL); GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].pool); @@ -3138,16 +3161,52 @@ bool WrappedVulkan::Serialise_vkCreateDevice(SerialiserType &ser, VkPhysicalDevi 1, }; - vkr = ObjDisp(device)->AllocateCommandBuffers(Unwrap(device), &cmdInfo, - &m_ExternalQueues[qidx].buffer); - RDCASSERTEQUAL(vkr, VK_SUCCESS); + VkFenceCreateInfo fenceInfo = {VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, NULL, + VK_FENCE_CREATE_SIGNALED_BIT}; + VkSemaphoreCreateInfo semInfo = {VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO}; - if(m_SetDeviceLoaderData) - m_SetDeviceLoaderData(device, m_ExternalQueues[qidx].buffer); - else - SetDispatchTableOverMagicNumber(device, m_ExternalQueues[qidx].buffer); + for(size_t x = 0; x < ARRAY_COUNT(m_ExternalQueues[i].ring); x++) + { + vkr = ObjDisp(device)->AllocateCommandBuffers(Unwrap(device), &cmdInfo, + &m_ExternalQueues[qidx].ring[x].acquire); + RDCASSERTEQUAL(vkr, VK_SUCCESS); - GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].buffer); + if(m_SetDeviceLoaderData) + m_SetDeviceLoaderData(device, m_ExternalQueues[qidx].ring[x].acquire); + else + SetDispatchTableOverMagicNumber(device, m_ExternalQueues[qidx].ring[x].acquire); + + GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].acquire); + + vkr = ObjDisp(device)->AllocateCommandBuffers(Unwrap(device), &cmdInfo, + &m_ExternalQueues[qidx].ring[x].release); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + if(m_SetDeviceLoaderData) + m_SetDeviceLoaderData(device, m_ExternalQueues[qidx].ring[x].release); + else + SetDispatchTableOverMagicNumber(device, m_ExternalQueues[qidx].ring[x].release); + + GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].release); + + vkr = ObjDisp(device)->CreateSemaphore(Unwrap(device), &semInfo, NULL, + &m_ExternalQueues[qidx].ring[x].fromext); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].fromext); + + vkr = ObjDisp(device)->CreateSemaphore(Unwrap(device), &semInfo, NULL, + &m_ExternalQueues[qidx].ring[x].toext); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].toext); + + vkr = ObjDisp(device)->CreateFence(Unwrap(device), &fenceInfo, NULL, + &m_ExternalQueues[qidx].ring[x].fence); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].fence); + } } ObjDisp(physicalDevice) @@ -3537,16 +3596,52 @@ VkResult WrappedVulkan::vkCreateDevice(VkPhysicalDevice physicalDevice, 1, }; - vkr = ObjDisp(device)->AllocateCommandBuffers(Unwrap(device), &cmdInfo, - &m_ExternalQueues[qidx].buffer); - RDCASSERTEQUAL(vkr, VK_SUCCESS); + VkFenceCreateInfo fenceInfo = {VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, NULL, + VK_FENCE_CREATE_SIGNALED_BIT}; + VkSemaphoreCreateInfo semInfo = {VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO}; - if(m_SetDeviceLoaderData) - m_SetDeviceLoaderData(device, m_ExternalQueues[qidx].buffer); - else - SetDispatchTableOverMagicNumber(device, m_ExternalQueues[qidx].buffer); + for(size_t x = 0; x < ARRAY_COUNT(m_ExternalQueues[i].ring); x++) + { + vkr = ObjDisp(device)->AllocateCommandBuffers(Unwrap(device), &cmdInfo, + &m_ExternalQueues[qidx].ring[x].acquire); + RDCASSERTEQUAL(vkr, VK_SUCCESS); - GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].buffer); + if(m_SetDeviceLoaderData) + m_SetDeviceLoaderData(device, m_ExternalQueues[qidx].ring[x].acquire); + else + SetDispatchTableOverMagicNumber(device, m_ExternalQueues[qidx].ring[x].acquire); + + GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].acquire); + + vkr = ObjDisp(device)->AllocateCommandBuffers(Unwrap(device), &cmdInfo, + &m_ExternalQueues[qidx].ring[x].release); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + if(m_SetDeviceLoaderData) + m_SetDeviceLoaderData(device, m_ExternalQueues[qidx].ring[x].release); + else + SetDispatchTableOverMagicNumber(device, m_ExternalQueues[qidx].ring[x].release); + + GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].release); + + vkr = ObjDisp(device)->CreateSemaphore(Unwrap(device), &semInfo, NULL, + &m_ExternalQueues[qidx].ring[x].fromext); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].fromext); + + vkr = ObjDisp(device)->CreateSemaphore(Unwrap(device), &semInfo, NULL, + &m_ExternalQueues[qidx].ring[x].toext); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].toext); + + vkr = ObjDisp(device)->CreateFence(Unwrap(device), &fenceInfo, NULL, + &m_ExternalQueues[qidx].ring[x].fence); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + GetResourceManager()->WrapResource(Unwrap(device), m_ExternalQueues[qidx].ring[x].fence); + } } ObjDisp(physicalDevice) @@ -3586,6 +3681,10 @@ void WrappedVulkan::vkDestroyDevice(VkDevice device, const VkAllocationCallbacks SubmitSemaphores(); FlushQ(); + // idle the device as well so that external queues are idle. + VkResult vkr = ObjDisp(m_Device)->DeviceWaitIdle(Unwrap(m_Device)); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + // MULTIDEVICE this function will need to check if the device is the one we // used for debugmanager/cmd pool etc, and only remove child queues and // resources (instead of doing full resource manager shutdown). @@ -3621,9 +3720,25 @@ void WrappedVulkan::vkDestroyDevice(VkDevice device, const VkAllocationCallbacks for(size_t i = 0; i < m_ExternalQueues.size(); i++) { - if(m_ExternalQueues[i].buffer != VK_NULL_HANDLE) + if(m_ExternalQueues[i].pool != VK_NULL_HANDLE) { - GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].buffer); + for(size_t x = 0; x < ARRAY_COUNT(m_ExternalQueues[i].ring); x++) + { + GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].acquire); + GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].release); + + ObjDisp(m_Device)->DestroySemaphore(Unwrap(m_Device), + Unwrap(m_ExternalQueues[i].ring[x].fromext), NULL); + GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].fromext); + + ObjDisp(m_Device)->DestroySemaphore(Unwrap(m_Device), + Unwrap(m_ExternalQueues[i].ring[x].toext), NULL); + GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].toext); + + ObjDisp(m_Device)->DestroyFence(Unwrap(m_Device), Unwrap(m_ExternalQueues[i].ring[x].fence), + NULL); + GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].ring[x].fence); + } ObjDisp(m_Device)->DestroyCommandPool(Unwrap(m_Device), Unwrap(m_ExternalQueues[i].pool), NULL); GetResourceManager()->ReleaseWrappedResource(m_ExternalQueues[i].pool); diff --git a/renderdoc/driver/vulkan/wrappers/vk_misc_funcs.cpp b/renderdoc/driver/vulkan/wrappers/vk_misc_funcs.cpp index ceaacb2e2..ae51718d9 100644 --- a/renderdoc/driver/vulkan/wrappers/vk_misc_funcs.cpp +++ b/renderdoc/driver/vulkan/wrappers/vk_misc_funcs.cpp @@ -243,6 +243,8 @@ void WrappedVulkan::vkDestroySwapchainKHR(VkDevice device, VkSwapchainKHR obj, { SwapchainInfo &info = *GetRecord(obj)->swapInfo; + ObjDisp(device)->DeviceWaitIdle(Unwrap(device)); + VkRenderPass unwrappedRP = Unwrap(info.rp); GetResourceManager()->ReleaseWrappedResource(info.rp, true); ObjDisp(device)->DestroyRenderPass(Unwrap(device), unwrappedRP, NULL); @@ -251,12 +253,18 @@ void WrappedVulkan::vkDestroySwapchainKHR(VkDevice device, VkSwapchainKHR obj, { VkFramebuffer unwrappedFB = Unwrap(info.images[i].fb); VkImageView unwrappedView = Unwrap(info.images[i].view); + VkSemaphore unwrappedSem = Unwrap(info.images[i].overlaydone); GetResourceManager()->ReleaseWrappedResource(info.images[i].fb, true); // note, image doesn't have to be destroyed, just untracked GetResourceManager()->ReleaseWrappedResource(info.images[i].im, true); GetResourceManager()->ReleaseWrappedResource(info.images[i].view, true); + GetResourceManager()->ReleaseWrappedResource(info.images[i].overlaydone); ObjDisp(device)->DestroyFramebuffer(Unwrap(device), unwrappedFB, NULL); ObjDisp(device)->DestroyImageView(Unwrap(device), unwrappedView, NULL); + ObjDisp(device)->DestroySemaphore(Unwrap(device), unwrappedSem, NULL); + + // return the command buffers to the pool + AddFreeCommandBuffer(info.images[i].cmd); } } diff --git a/renderdoc/driver/vulkan/wrappers/vk_wsi_funcs.cpp b/renderdoc/driver/vulkan/wrappers/vk_wsi_funcs.cpp index c6a2e2b1a..8f1187be5 100644 --- a/renderdoc/driver/vulkan/wrappers/vk_wsi_funcs.cpp +++ b/renderdoc/driver/vulkan/wrappers/vk_wsi_funcs.cpp @@ -583,6 +583,21 @@ void WrappedVulkan::WrapAndProcessCreatedSwapchain(VkDevice device, state->isMemoryBound = true; } + // take a command buffer and steal it + swapImInfo.cmd = GetNextCmd(); + RemovePendingCommandBuffer(swapImInfo.cmd); + + { + VkSemaphoreCreateInfo semInfo = {VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO}; + + vkr = ObjDisp(device)->CreateSemaphore(Unwrap(device), &semInfo, NULL, + &swapImInfo.overlaydone); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + GetResourceManager()->WrapResource(Unwrap(device), swapImInfo.overlaydone); + GetResourceManager()->SetInternalResource(GetResID(swapImInfo.overlaydone)); + } + { VkImageViewCreateInfo info = { VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, @@ -714,8 +729,8 @@ VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR for(uint32_t i = 0; i < unwrappedInfo.waitSemaphoreCount; i++) unwrappedSems.push_back(Unwrap(unwrappedInfo.pWaitSemaphores[i])); - unwrappedInfo.pSwapchains = unwrappedInfo.swapchainCount ? &unwrappedSwaps[0] : NULL; - unwrappedInfo.pWaitSemaphores = unwrappedInfo.waitSemaphoreCount ? &unwrappedSems[0] : NULL; + unwrappedInfo.pSwapchains = unwrappedSwaps.data(); + unwrappedInfo.pWaitSemaphores = unwrappedSems.data(); // Don't support any extensions for present info const VkBaseInStructure *next = (const VkBaseInStructure *)pPresentInfo->pNext; @@ -764,6 +779,8 @@ VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR VkRenderPass rp = swapInfo.rp; VkImage im = swapInfo.images[pPresentInfo->pImageIndices[0]].im; VkFramebuffer fb = swapInfo.images[pPresentInfo->pImageIndices[0]].fb; + VkCommandBuffer cmd = swapInfo.images[pPresentInfo->pImageIndices[0]].cmd; + VkSemaphore sem = swapInfo.images[pPresentInfo->pImageIndices[0]].overlaydone; VkResourceRecord *queueRecord = GetRecord(queue); uint32_t swapQueueIndex = queueRecord->queueFamilyIndex; @@ -771,7 +788,7 @@ VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR VkDevDispatchTable *vt = ObjDisp(GetDev()); TextPrintState textstate = { - GetNextCmd(), + cmd, rp, fb, RDCMAX(1U, swapInfo.imageInfo.extent.width), @@ -779,10 +796,12 @@ VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR swapInfo.imageInfo.format, }; + ObjDisp(cmd)->ResetCommandBuffer(Unwrap(cmd), 0); + VkCommandBufferBeginInfo beginInfo = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, NULL, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; - VkResult vkr = vt->BeginCommandBuffer(Unwrap(textstate.cmd), &beginInfo); + VkResult vkr = vt->BeginCommandBuffer(Unwrap(cmd), &beginInfo); RDCASSERTEQUAL(vkr, VK_SUCCESS); VkImageMemoryBarrier bbBarrier = { @@ -804,11 +823,35 @@ VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR bbBarrier.srcAccessMask = VK_ACCESS_ALL_READ_BITS; bbBarrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; - DoPipelineBarrier(textstate.cmd, 1, &bbBarrier); + DoPipelineBarrier(cmd, 1, &bbBarrier); + + rdcarray waitStage; + waitStage.fill(unwrappedSems.size(), VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); + + uint32_t ringIdx = 0; + + VkSubmitInfo submitInfo = {VK_STRUCTURE_TYPE_SUBMIT_INFO}; + + // wait on the present's semaphores + submitInfo.pWaitDstStageMask = waitStage.data(); + submitInfo.pWaitSemaphores = unwrappedSems.data(); + submitInfo.waitSemaphoreCount = (uint32_t)unwrappedSems.size(); + + // and signal overlaydone + submitInfo.pSignalSemaphores = UnwrapPtr(sem); + submitInfo.signalSemaphoreCount = 1; if(swapQueueIndex != m_QueueFamilyIdx) { - VkCommandBuffer extQCmd = GetExtQueueCmd(swapQueueIndex); + ringIdx = m_ExternalQueues[swapQueueIndex].GetNextIdx(); + + VkCommandBuffer extQCmd = m_ExternalQueues[swapQueueIndex].ring[ringIdx].acquire; + VkFence fence = m_ExternalQueues[swapQueueIndex].ring[ringIdx].fence; + + // wait for this queue ring to be free, but with a reasonably aggressive timeout (50ms). If + // this ring has never been used the fence is signalled on creation + vkr = vt->WaitForFences(Unwrap(m_Device), 1, UnwrapPtr(fence), VK_TRUE, 50000000); + RDCASSERTEQUAL(vkr, VK_SUCCESS); vkr = vt->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo); RDCASSERTEQUAL(vkr, VK_SUCCESS); @@ -817,7 +860,33 @@ VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd)); - SubmitAndFlushExtQueue(swapQueueIndex); + VkQueue q = m_ExternalQueues[swapQueueIndex].queue; + + // wait on present semaphores as before, but on the proper (external) queue. + + // submit the acquire command buffer (release from the external point of view) + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = UnwrapPtr(extQCmd); + + // signal the semaphore that we'll wait on for our overlay command buffer + submitInfo.signalSemaphoreCount = 1; + submitInfo.pSignalSemaphores = + UnwrapPtr(m_ExternalQueues[swapQueueIndex].ring[ringIdx].fromext); + + vkr = ObjDisp(q)->QueueSubmit(Unwrap(q), 1, &submitInfo, VK_NULL_HANDLE); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + + // next submit needs to wait on fromext + unwrappedSems.assign(submitInfo.pSignalSemaphores, 1); + waitStage.resize(1); + + submitInfo.pWaitDstStageMask = waitStage.data(); + submitInfo.pWaitSemaphores = unwrappedSems.data(); + submitInfo.waitSemaphoreCount = (uint32_t)unwrappedSems.size(); + + // and signal toext + submitInfo.pSignalSemaphores = + UnwrapPtr(m_ExternalQueues[swapQueueIndex].ring[ringIdx].toext); } int flags = activeWindow ? RenderDoc::eOverlay_ActiveWindow : 0; @@ -839,15 +908,20 @@ VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR DoPipelineBarrier(textstate.cmd, 1, &bbBarrier); - ObjDisp(textstate.cmd)->EndCommandBuffer(Unwrap(textstate.cmd)); + ObjDisp(cmd)->EndCommandBuffer(Unwrap(cmd)); - rdcarray waitStage; - waitStage.fill(unwrappedSems.size(), VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); - SubmitCmds(unwrappedSems.data(), waitStage.data(), (uint32_t)unwrappedSems.size()); + { + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = UnwrapPtr(cmd); + + vkr = ObjDisp(m_Queue)->QueueSubmit(Unwrap(m_Queue), 1, &submitInfo, VK_NULL_HANDLE); + RDCASSERTEQUAL(vkr, VK_SUCCESS); + } if(swapQueueIndex != m_QueueFamilyIdx) { - VkCommandBuffer extQCmd = GetExtQueueCmd(swapQueueIndex); + VkCommandBuffer extQCmd = m_ExternalQueues[swapQueueIndex].ring[ringIdx].release; + VkFence fence = m_ExternalQueues[swapQueueIndex].ring[ringIdx].fence; vkr = vt->BeginCommandBuffer(Unwrap(extQCmd), &beginInfo); RDCASSERTEQUAL(vkr, VK_SUCCESS); @@ -856,10 +930,33 @@ VkResult WrappedVulkan::vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR ObjDisp(extQCmd)->EndCommandBuffer(Unwrap(extQCmd)); - SubmitAndFlushExtQueue(swapQueueIndex); + VkQueue q = m_ExternalQueues[swapQueueIndex].queue; + + // wait on toext which was signalled above + unwrappedSems.assign(submitInfo.pSignalSemaphores, 1); + waitStage.resize(1); + + submitInfo.pWaitDstStageMask = waitStage.data(); + submitInfo.pWaitSemaphores = unwrappedSems.data(); + submitInfo.waitSemaphoreCount = (uint32_t)unwrappedSems.size(); + + // release to the external queue + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = UnwrapPtr(extQCmd); + + // signal the overlaydone semaphore + submitInfo.signalSemaphoreCount = 1; + submitInfo.pSignalSemaphores = UnwrapPtr(sem); + + // submit and signal the fence when we're done, so we know next time around that this is + // safe to re-use + vkr = ObjDisp(q)->QueueSubmit(Unwrap(q), 1, &submitInfo, Unwrap(fence)); + RDCASSERTEQUAL(vkr, VK_SUCCESS); } - FlushQ(); + // the present waits on our new semaphore + unwrappedInfo.waitSemaphoreCount = 1; + unwrappedInfo.pWaitSemaphores = submitInfo.pSignalSemaphores; } GetResourceManager()->CleanBackgroundFrameReferences();