From bc10ce84ba11da69358d63fec63b470d826286db Mon Sep 17 00:00:00 2001 From: baldurk Date: Thu, 17 Dec 2015 14:31:10 +0100 Subject: [PATCH] Use semaphores to synchronise unbind/bind of sparse mappings * For non-image mappings (image opaque and buffer) we unbind the whole resource then bind the initial mapping. This means two batches though so needs to be synchronised. --- renderdoc/driver/vulkan/vk_core.cpp | 37 ++++++++++++++++ renderdoc/driver/vulkan/vk_core.h | 25 ++++++++--- renderdoc/driver/vulkan/vk_initstate.cpp | 44 +++++++++++++++---- .../vulkan/wrappers/vk_device_funcs.cpp | 9 +++- 4 files changed, 99 insertions(+), 16 deletions(-) diff --git a/renderdoc/driver/vulkan/vk_core.cpp b/renderdoc/driver/vulkan/vk_core.cpp index 4fddbf5fc..66c08406a 100644 --- a/renderdoc/driver/vulkan/vk_core.cpp +++ b/renderdoc/driver/vulkan/vk_core.cpp @@ -406,6 +406,43 @@ void WrappedVulkan::SubmitCmds() m_InternalCmds.pendingcmds.clear(); } +VkSemaphore WrappedVulkan::GetNextSemaphore() +{ + VkSemaphore ret; + + if(!m_InternalCmds.freesems.empty()) + { + ret = m_InternalCmds.freesems.back(); + m_InternalCmds.freesems.pop_back(); + + // assume semaphore is back to unsignaled state after being waited on + } + else + { + VkSemaphoreCreateInfo semInfo = { VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO }; + VkResult vkr = ObjDisp(m_Device)->CreateSemaphore(Unwrap(m_Device), &semInfo, NULL, &ret); + RDCASSERT(vkr == VK_SUCCESS); + + GetResourceManager()->WrapResource(Unwrap(m_Device), ret); + } + + m_InternalCmds.pendingsems.push_back(ret); + + return ret; +} + +void WrappedVulkan::SubmitSemaphores() +{ + // nothing to do + if(m_InternalCmds.pendingsems.empty()) + return; + + // no actual submission, just mark them as 'done with' so they will be + // recycled on next flush + m_InternalCmds.submittedsems.insert(m_InternalCmds.submittedsems.end(), m_InternalCmds.pendingsems.begin(), m_InternalCmds.pendingsems.end()); + m_InternalCmds.pendingsems.clear(); +} + void WrappedVulkan::FlushQ() { // VKTODOLOW could do away with the need for this function by keeping diff --git a/renderdoc/driver/vulkan/vk_core.h b/renderdoc/driver/vulkan/vk_core.h index 9afcdd0e0..061510d99 100644 --- a/renderdoc/driver/vulkan/vk_core.h +++ b/renderdoc/driver/vulkan/vk_core.h @@ -207,16 +207,27 @@ private: freecmds.clear(); pendingcmds.clear(); submittedcmds.clear(); + + freecmds.clear(); + pendingcmds.clear(); + submittedcmds.clear(); } VkCommandPool m_CmdPool; // the command pool used for allocating our own command buffers - vector freecmds; - // -> record -> - vector pendingcmds; - // -> submit -> - vector submittedcmds; - // -> flush/waitidle -> freecmds + vector freecmds; // < + // -> GetNextCmd() -> // | + vector pendingcmds; // | + // -> SubmitCmds() -> | + vector submittedcmds; // | + // -> FlushQ() ----------------------------^ + + vector freesems; // < + // -> GetNextSemaphore() -> // | + vector pendingsems; // | + // -> SubmitSemaphores() -> | + vector submittedsems; // | + // -> FlushQ() -----------------------^ } m_InternalCmds; vector m_CleanupMems; @@ -229,6 +240,8 @@ private: VkPhysicalDevice GetPhysDev() { RDCASSERT(m_PhysicalDevice != VK_NULL_HANDLE); return m_PhysicalDevice; } VkCommandBuffer GetNextCmd(); void SubmitCmds(); + VkSemaphore GetNextSemaphore(); + void SubmitSemaphores(); void FlushQ(); const VkPhysicalDeviceFeatures &GetDeviceFeatures() diff --git a/renderdoc/driver/vulkan/vk_initstate.cpp b/renderdoc/driver/vulkan/vk_initstate.cpp index 1ee0a433a..3583837cb 100644 --- a/renderdoc/driver/vulkan/vk_initstate.cpp +++ b/renderdoc/driver/vulkan/vk_initstate.cpp @@ -722,14 +722,19 @@ bool WrappedVulkan::Apply_SparseInitialState(WrappedVkBuffer *buf, VulkanResourc buf->real.As(), 1, &unbind }; - // VKTODOLOW need to signal/wait semaphore between the unbind and bind + VkDevice d = GetDev(); + + // this semaphore separates the unbind and bind, as there isn't an ordering guarantee + // for two adjacent batches that bind the same resource. + VkSemaphore sem = GetNextSemaphore(); + VkBindSparseInfo bindsparse = { VK_STRUCTURE_TYPE_BIND_SPARSE_INFO, NULL, 0, NULL, // wait semaphores 1, &bufBind, 0, NULL, // image opaque 0, NULL, // image bind - 0, NULL, // signal semaphores + 1, UnwrapPtr(sem), // signal semaphores }; // first unbind all @@ -740,17 +745,25 @@ bool WrappedVulkan::Apply_SparseInitialState(WrappedVkBuffer *buf, VulkanResourc { bufBind.bindCount = info->numBinds; bufBind.pBinds = info->binds; + + // wait for unbind semaphore + bindsparse.waitSemaphoreCount = 1; + bindsparse.pWaitSemaphores = bindsparse.pSignalSemaphores; + + bindsparse.signalSemaphoreCount = 0; + bindsparse.pSignalSemaphores = NULL; - // VKTODOLOW need to signal/wait semaphore after bind somehow ObjDisp(q)->QueueBindSparse(Unwrap(q), 1, &bindsparse, VK_NULL_HANDLE); } + + // marks that the above semaphore has been used, so next time we + // flush it will be moved back to the pool + SubmitSemaphores(); VkResult vkr = VK_SUCCESS; VkBuffer srcBuf = (VkBuffer)(uint64_t)contents.resource; - VkDevice d = GetDev(); - VkCommandBuffer cmd = GetNextCmd(); VkCommandBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, NULL, VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT }; @@ -791,6 +804,8 @@ bool WrappedVulkan::Apply_SparseInitialState(WrappedVkBuffer *buf, VulkanResourc vkr = ObjDisp(cmd)->EndCommandBuffer(Unwrap(cmd)); RDCASSERT(vkr == VK_SUCCESS); + FlushQ(); + return true; } @@ -818,15 +833,16 @@ bool WrappedVulkan::Apply_SparseInitialState(WrappedVkImage *im, VulkanResourceM VkSparseImageOpaqueMemoryBindInfo opaqueBind = { im->real.As(), 1, &unbind }; + + VkSemaphore sem = GetNextSemaphore(); - // VKTODOLOW need to signal/wait semaphore between the unbind and bind VkBindSparseInfo bindsparse = { VK_STRUCTURE_TYPE_BIND_SPARSE_INFO, NULL, 0, NULL, // wait semaphores 0, NULL, // buffer bind 1, &opaqueBind, 0, NULL, // image bind - 0, NULL, // signal semaphores + 1, UnwrapPtr(sem), // signal semaphores }; // first unbind all @@ -837,15 +853,25 @@ bool WrappedVulkan::Apply_SparseInitialState(WrappedVkImage *im, VulkanResourceM { opaqueBind.bindCount = info->opaqueCount; opaqueBind.pBinds = info->opaque; - // VKTODOLOW need to signal/wait semaphore after bind somehow + + // wait for unbind semaphore + bindsparse.waitSemaphoreCount = 1; + bindsparse.pWaitSemaphores = bindsparse.pSignalSemaphores; + + bindsparse.signalSemaphoreCount = 0; + bindsparse.pSignalSemaphores = NULL; + ObjDisp(q)->QueueBindSparse(Unwrap(q), 1, &bindsparse, VK_NULL_HANDLE); } + + // marks that the above semaphore has been used, so next time we + // flush it will be moved back to the pool + SubmitSemaphores(); } { VkSparseImageMemoryBindInfo imgBinds[NUM_VK_IMAGE_ASPECTS] = { 0 }; - // VKTODOLOW need to signal/wait semaphore after bind somehow VkBindSparseInfo bindsparse = { VK_STRUCTURE_TYPE_BIND_SPARSE_INFO, NULL, 0, NULL, // wait semaphores diff --git a/renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp b/renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp index 73a012efd..930e8fa7a 100644 --- a/renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp +++ b/renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp @@ -759,8 +759,9 @@ VkResult WrappedVulkan::vkCreateDevice( void WrappedVulkan::vkDestroyDevice(VkDevice device, const VkAllocationCallbacks* pAllocator) { - // flush out any pending commands + // flush out any pending commands/semaphores SubmitCmds(); + SubmitSemaphores(); FlushQ(); // MULTIDEVICE this function will need to check if the device is the one we @@ -784,6 +785,12 @@ void WrappedVulkan::vkDestroyDevice(VkDevice device, const VkAllocationCallbacks ObjDisp(m_Device)->DestroyCommandPool(Unwrap(m_Device), Unwrap(m_InternalCmds.m_CmdPool), NULL); GetResourceManager()->ReleaseWrappedResource(m_InternalCmds.m_CmdPool); } + + for(size_t i=0; i < m_InternalCmds.freesems.size(); i++) + { + ObjDisp(m_Device)->DestroySemaphore(Unwrap(m_Device), Unwrap(m_InternalCmds.freesems[i]), NULL); + GetResourceManager()->ReleaseWrappedResource(m_InternalCmds.freesems[i]); + } m_InternalCmds.Reset();