diff --git a/renderdoc/driver/vulkan/vk_core.cpp b/renderdoc/driver/vulkan/vk_core.cpp index 53450bf5a..e95a40435 100644 --- a/renderdoc/driver/vulkan/vk_core.cpp +++ b/renderdoc/driver/vulkan/vk_core.cpp @@ -447,6 +447,67 @@ WrappedVulkan::~WrappedVulkan() SAFE_DELETE(m_pSerialiser); } +VkCmdBuffer WrappedVulkan::GetNextCmd() +{ + RDCASSERT(m_SwapPhysDevice >= 0); + ReplayData &rd = m_PhysicalReplayData[m_SwapPhysDevice]; + + VkCmdBuffer ret; + + if(!rd.freecmds.empty()) + { + ret = rd.freecmds.back(); + rd.freecmds.pop_back(); + } + else + { + VkCmdBufferCreateInfo cmdInfo = { VK_STRUCTURE_TYPE_CMD_BUFFER_CREATE_INFO, NULL, Unwrap(rd.cmdpool), VK_CMD_BUFFER_LEVEL_PRIMARY, 0 }; + VkResult vkr = ObjDisp(rd.dev)->CreateCommandBuffer(Unwrap(rd.dev), &cmdInfo, &ret); + RDCASSERT(vkr == VK_SUCCESS); + + GetResourceManager()->WrapResource(Unwrap(rd.dev), ret); + } + + rd.pendingcmds.push_back(ret); + + return ret; +} + +void WrappedVulkan::SubmitCmds() +{ + RDCASSERT(m_SwapPhysDevice >= 0); + ReplayData &rd = m_PhysicalReplayData[m_SwapPhysDevice]; + + // nothing to do + if(rd.pendingcmds.empty()) + return; + + vector cmds = rd.pendingcmds; + for(size_t i=0; i < cmds.size(); i++) cmds[i] = Unwrap(cmds[i]); + + ObjDisp(rd.q)->QueueSubmit(Unwrap(rd.q), (uint32_t)cmds.size(), &cmds[0], VK_NULL_HANDLE); + + rd.submittedcmds.insert(rd.submittedcmds.end(), rd.pendingcmds.begin(), rd.pendingcmds.end()); + rd.pendingcmds.clear(); +} + +void WrappedVulkan::FlushQ() +{ + RDCASSERT(m_SwapPhysDevice >= 0); + ReplayData &rd = m_PhysicalReplayData[m_SwapPhysDevice]; + + // VKTODOLOW could do away with the need for this function by keeping + // commands until N presents later, or something, or checking on fences + + ObjDisp(rd.q)->QueueWaitIdle(Unwrap(rd.q)); + + if(!rd.submittedcmds.empty()) + { + rd.freecmds.insert(rd.freecmds.end(), rd.submittedcmds.begin(), rd.submittedcmds.end()); + rd.submittedcmds.clear(); + } +} + const char * WrappedVulkan::GetChunkName(uint32_t idx) { if(idx < FIRST_CHUNK_ID || idx >= NUM_VULKAN_CHUNKS) @@ -601,8 +662,7 @@ bool WrappedVulkan::Serialise_BeginCaptureFrame(bool applyInitialState) if(applyInitialState && !imgTransitions.empty()) { - VkCmdBuffer cmd = GetCmd(); - VkQueue q = GetQ(); + VkCmdBuffer cmd = GetNextCmd(); VkCmdBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_CMD_BUFFER_BEGIN_INFO, NULL, VK_CMD_BUFFER_OPTIMIZE_SMALL_BATCH_BIT | VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT }; @@ -624,12 +684,9 @@ bool WrappedVulkan::Serialise_BeginCaptureFrame(bool applyInitialState) vkr = ObjDisp(cmd)->EndCommandBuffer(Unwrap(cmd)); RDCASSERT(vkr == VK_SUCCESS); - vkr = ObjDisp(q)->QueueSubmit(Unwrap(q), 1, UnwrapPtr(cmd), VK_NULL_HANDLE); - RDCASSERT(vkr == VK_SUCCESS); - // VKTODOMED while we're reusing cmd buffer, we have to ensure this one - // is done before continuing - vkr = ObjDisp(q)->QueueWaitIdle(Unwrap(q)); - RDCASSERT(vkr == VK_SUCCESS); + + SubmitCmds(); + // don't need to flush here } return true; @@ -768,7 +825,6 @@ void WrappedVulkan::ReadLogInitialisation() RDCASSERT(m_SwapPhysDevice >= 0 && m_PhysicalReplayData[m_SwapPhysDevice].dev != VK_NULL_HANDLE && m_PhysicalReplayData[m_SwapPhysDevice].q != VK_NULL_HANDLE && - m_PhysicalReplayData[m_SwapPhysDevice].cmd != VK_NULL_HANDLE && m_PhysicalReplayData[m_SwapPhysDevice].cmdpool != VK_NULL_HANDLE); // VKTODOLOW maybe better place to put this? @@ -1334,8 +1390,7 @@ void WrappedVulkan::ReplayLog(uint32_t frameID, uint32_t startEventID, uint32_t if(m_FakeBBImgId != ResourceId()) { VkDevice dev = GetDev(); - VkCmdBuffer cmd = GetCmd(); - VkQueue q = GetQ(); + VkCmdBuffer cmd = GetNextCmd(); VkCmdBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_CMD_BUFFER_BEGIN_INFO, NULL, VK_CMD_BUFFER_OPTIMIZE_SMALL_BATCH_BIT | VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT }; @@ -1370,12 +1425,8 @@ void WrappedVulkan::ReplayLog(uint32_t frameID, uint32_t startEventID, uint32_t vkr = ObjDisp(cmd)->EndCommandBuffer(Unwrap(cmd)); RDCASSERT(vkr == VK_SUCCESS); - vkr = ObjDisp(q)->QueueSubmit(Unwrap(q), 1, UnwrapPtr(cmd), VK_NULL_HANDLE); - RDCASSERT(vkr == VK_SUCCESS); - // VKTODOMED while we're reusing cmd buffer, we have to ensure this one - // is done before continuing - vkr = ObjDisp(q)->QueueWaitIdle(Unwrap(q)); - RDCASSERT(vkr == VK_SUCCESS); + SubmitCmds(); + // don't need to flush here } } diff --git a/renderdoc/driver/vulkan/vk_core.h b/renderdoc/driver/vulkan/vk_core.h index e168d1e79..227dc0341 100644 --- a/renderdoc/driver/vulkan/vk_core.h +++ b/renderdoc/driver/vulkan/vk_core.h @@ -163,14 +163,21 @@ private: ReplayData() : inst(VK_NULL_HANDLE), phys(VK_NULL_HANDLE) , qFamilyIdx(0), dev(VK_NULL_HANDLE), q(VK_NULL_HANDLE) - , cmd(VK_NULL_HANDLE), cmdpool(VK_NULL_HANDLE), debugMan(NULL) {} + , cmdpool(VK_NULL_HANDLE), debugMan(NULL) {} VkInstance inst; VkPhysicalDevice phys; VkDevice dev; uint32_t qFamilyIdx; VkQueue q; - VkCmdBuffer cmd; + + vector freecmds; + // -> record -> + vector pendingcmds; + // -> submit -> + vector submittedcmds; + // -> flush/waitidle -> freecmds + VkCmdPool cmdpool; VulkanDebugManager *debugMan; @@ -196,8 +203,11 @@ private: { RDCASSERT(m_SwapPhysDevice >= 0); return m_PhysicalReplayData[m_SwapPhysDevice].debugMan; } VkDevice GetDev() { RDCASSERT(m_SwapPhysDevice >= 0); return m_PhysicalReplayData[m_SwapPhysDevice].dev; } - VkQueue GetQ() { RDCASSERT(m_SwapPhysDevice >= 0); return m_PhysicalReplayData[m_SwapPhysDevice].q; } - VkCmdBuffer GetCmd(){ RDCASSERT(m_SwapPhysDevice >= 0); return m_PhysicalReplayData[m_SwapPhysDevice].cmd; } + VkQueue GetQ() { RDCASSERT(m_SwapPhysDevice >= 0); return m_PhysicalReplayData[m_SwapPhysDevice].q; } + VkCmdBuffer GetNextCmd(); + void SubmitCmds(); + void FlushQ(); + uint32_t GetReadbackMemoryIndex(uint32_t resourceRequiredBitmask); uint32_t GetUploadMemoryIndex(uint32_t resourceRequiredBitmask); uint32_t GetGPULocalMemoryIndex(uint32_t resourceRequiredBitmask); diff --git a/renderdoc/driver/vulkan/vk_debug.cpp b/renderdoc/driver/vulkan/vk_debug.cpp index b974d4d7c..91ec4018f 100644 --- a/renderdoc/driver/vulkan/vk_debug.cpp +++ b/renderdoc/driver/vulkan/vk_debug.cpp @@ -623,6 +623,15 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver, VkDevice dev) VKMGR()->ReleaseWrappedResource(module[i]); } + VkCmdBuffer cmd = driver->GetNextCmd(); + + VkCmdBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_CMD_BUFFER_BEGIN_INFO, NULL, VK_CMD_BUFFER_OPTIMIZE_SMALL_BATCH_BIT | VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT }; + + vkr = vt->ResetCommandBuffer(Unwrap(cmd), 0); + RDCASSERT(vkr == VK_SUCCESS); + vkr = vt->BeginCommandBuffer(Unwrap(cmd), &beginInfo); + RDCASSERT(vkr == VK_SUCCESS); + { int width = FONT_TEX_WIDTH, height = FONT_TEX_HEIGHT; @@ -707,15 +716,6 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver, VkDevice dev) VKMGR()->WrapResource(Unwrap(dev), m_TextAtlasView); // need to transition image into valid state, then upload - VkCmdBuffer cmd = driver->GetCmd(); - VkQueue q = driver->GetQ(); - - VkCmdBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_CMD_BUFFER_BEGIN_INFO, NULL, VK_CMD_BUFFER_OPTIMIZE_SMALL_BATCH_BIT | VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT }; - - vkr = vt->ResetCommandBuffer(Unwrap(cmd), 0); - RDCASSERT(vkr == VK_SUCCESS); - vkr = vt->BeginCommandBuffer(Unwrap(cmd), &beginInfo); - RDCASSERT(vkr == VK_SUCCESS); VkImageMemoryBarrier trans = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, NULL, @@ -729,15 +729,6 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver, VkDevice dev) vt->CmdPipelineBarrier(Unwrap(cmd), VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, false, 1, &barrier); - vt->EndCommandBuffer(Unwrap(cmd)); - - vt->QueueSubmit(Unwrap(q), 1, UnwrapPtr(cmd), VK_NULL_HANDLE); - - // VKTODOMED ideally all the commands from Bind to Flip would be recorded - // into a single command buffer and we can just have several allocated - // ring-buffer style - vt->QueueWaitIdle(Unwrap(q)); - byte *pData = NULL; vkr = vt->MapMemory(Unwrap(dev), Unwrap(m_TextAtlasMem), 0, 0, 0, (void **)&pData); RDCASSERT(vkr == VK_SUCCESS); @@ -833,15 +824,6 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver, VkDevice dev) VKMGR()->WrapResource(Unwrap(dev), m_PickPixelImageView); // need to transition image into valid state - VkCmdBuffer cmd = driver->GetCmd(); - VkQueue q = driver->GetQ(); - - VkCmdBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_CMD_BUFFER_BEGIN_INFO, NULL, VK_CMD_BUFFER_OPTIMIZE_SMALL_BATCH_BIT | VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT }; - - vkr = vt->ResetCommandBuffer(Unwrap(cmd), 0); - RDCASSERT(vkr == VK_SUCCESS); - vkr = vt->BeginCommandBuffer(Unwrap(cmd), &beginInfo); - RDCASSERT(vkr == VK_SUCCESS); VkImageMemoryBarrier trans = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, NULL, @@ -856,15 +838,6 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver, VkDevice dev) vt->CmdPipelineBarrier(Unwrap(cmd), VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, false, 1, &barrier); - vt->EndCommandBuffer(Unwrap(cmd)); - - vt->QueueSubmit(Unwrap(q), 1, UnwrapPtr(cmd), VK_NULL_HANDLE); - - // VKTODOMED ideally all the commands from Bind to Flip would be recorded - // into a single command buffer and we can just have several allocated - // ring-buffer style - vt->QueueWaitIdle(Unwrap(q)); - // create render pass VkAttachmentDescription attDesc = { VK_STRUCTURE_TYPE_ATTACHMENT_DESCRIPTION, NULL, @@ -967,6 +940,9 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver, VkDevice dev) }; vt->UpdateDescriptorSets(Unwrap(dev), ARRAY_COUNT(writeSet), writeSet, 0, NULL); + + vt->EndCommandBuffer(Unwrap(cmd)); + driver->SubmitCmds(); } VulkanDebugManager::~VulkanDebugManager() @@ -1209,8 +1185,4 @@ void VulkanDebugManager::RenderTextInternal(const TextPrintState &textstate, flo } vt->EndCommandBuffer(Unwrap(textstate.cmd)); - - vt->QueueSubmit(Unwrap(textstate.q), 1, UnwrapPtr(textstate.cmd), VK_NULL_HANDLE); - - vt->QueueWaitIdle(Unwrap(textstate.q)); } diff --git a/renderdoc/driver/vulkan/vk_debug.h b/renderdoc/driver/vulkan/vk_debug.h index 70e93033f..7f27c906e 100644 --- a/renderdoc/driver/vulkan/vk_debug.h +++ b/renderdoc/driver/vulkan/vk_debug.h @@ -32,7 +32,6 @@ struct TextPrintState { - VkQueue q; VkCmdBuffer cmd; VkRenderPass rp; VkFramebuffer fb; diff --git a/renderdoc/driver/vulkan/vk_initstate.cpp b/renderdoc/driver/vulkan/vk_initstate.cpp index ddbc1f3ae..48b11c232 100644 --- a/renderdoc/driver/vulkan/vk_initstate.cpp +++ b/renderdoc/driver/vulkan/vk_initstate.cpp @@ -63,8 +63,10 @@ bool WrappedVulkan::Prepare_InitialState(WrappedVkRes *res) VkResult vkr = VK_SUCCESS; VkDevice d = GetDev(); - VkQueue q = GetQ(); - VkCmdBuffer cmd = GetCmd(); + // VKTODOLOW ideally the prepares could be batched up + // a bit more - maybe not all in one command buffer, but + // at least more than one each + VkCmdBuffer cmd = GetNextCmd(); VkDeviceMemory mem = VK_NULL_HANDLE; @@ -118,13 +120,12 @@ bool WrappedVulkan::Prepare_InitialState(WrappedVkRes *res) vkr = ObjDisp(d)->EndCommandBuffer(Unwrap(cmd)); RDCASSERT(vkr == VK_SUCCESS); - vkr = ObjDisp(d)->QueueSubmit(Unwrap(q), 1, UnwrapPtr(cmd), VK_NULL_HANDLE); - RDCASSERT(vkr == VK_SUCCESS); - - // VKTODOMED would be nice to store a fence too at this point - // so we can sync on that on serialise rather than syncing - // every time. - ObjDisp(d)->QueueWaitIdle(Unwrap(q)); + // VKTODOLOW would be nice to store up all these buffers so that + // we don't have to submit & flush here before destroying, but + // instead could submit all cmds, then flush once, then destroy + // buffers. (or even not flush at all until capture is over) + SubmitCmds(); + FlushQ(); ObjDisp(d)->DestroyBuffer(Unwrap(d), srcBuf); ObjDisp(d)->DestroyBuffer(Unwrap(d), dstBuf); @@ -482,8 +483,7 @@ void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, VulkanResourceManager VkResult vkr = VK_SUCCESS; VkDevice d = GetDev(); - VkQueue q = GetQ(); - VkCmdBuffer cmd = GetCmd(); + VkCmdBuffer cmd = GetNextCmd(); VkCmdBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_CMD_BUFFER_BEGIN_INFO, NULL, VK_CMD_BUFFER_OPTIMIZE_SMALL_BATCH_BIT | VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT }; @@ -516,13 +516,12 @@ void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, VulkanResourceManager vkr = ObjDisp(cmd)->EndCommandBuffer(Unwrap(cmd)); RDCASSERT(vkr == VK_SUCCESS); - vkr = ObjDisp(q)->QueueSubmit(Unwrap(q), 1, UnwrapPtr(cmd), VK_NULL_HANDLE); - RDCASSERT(vkr == VK_SUCCESS); - - // VKTODOMED would be nice to store a fence too at this point - // so we can sync on that on serialise rather than syncing - // every time. - ObjDisp(q)->QueueWaitIdle(Unwrap(q)); + // VKTODOLOW if this dstBuf was persistent or at least cached + // we could batch these command buffers better and wouldn't + // need to flush at all until application of all init states + // is over + SubmitCmds(); + FlushQ(); ObjDisp(d)->DestroyBuffer(Unwrap(d), dstBuf); } diff --git a/renderdoc/driver/vulkan/vk_replay.cpp b/renderdoc/driver/vulkan/vk_replay.cpp index 1116e2c8f..94dfcd085 100644 --- a/renderdoc/driver/vulkan/vk_replay.cpp +++ b/renderdoc/driver/vulkan/vk_replay.cpp @@ -566,8 +566,7 @@ void VulkanReplay::PickPixel(ResourceId texture, uint32_t x, uint32_t y, uint32_ } VkDevice dev = m_pDriver->GetDev(); - VkCmdBuffer cmd = m_pDriver->GetCmd(); - VkQueue q = m_pDriver->GetQ(); + VkCmdBuffer cmd = m_pDriver->GetNextCmd(); const VkLayerDispatchTable *vt = ObjDisp(dev); VkResult vkr = VK_SUCCESS; @@ -612,12 +611,12 @@ void VulkanReplay::PickPixel(ResourceId texture, uint32_t x, uint32_t y, uint32_ vt->CmdPipelineBarrier(Unwrap(cmd), VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, false, 1, &barrier); vt->EndCommandBuffer(Unwrap(cmd)); - - vt->QueueSubmit(Unwrap(q), 1, UnwrapPtr(cmd), VK_NULL_HANDLE); - - vt->QueueWaitIdle(Unwrap(q)); } + // submit cmds and wait for idle so we can readback + m_pDriver->SubmitCmds(); + m_pDriver->FlushQ(); + float *pData = NULL; vt->MapMemory(Unwrap(dev), Unwrap(GetDebugManager()->m_PickPixelReadbackBuffer.mem), 0, 0, 0, (void **)&pData); @@ -669,8 +668,7 @@ bool VulkanReplay::RenderTexture(TextureDisplay cfg) bool VulkanReplay::RenderTextureInternal(TextureDisplay cfg, VkRenderPassBeginInfo rpbegin, bool blendAlpha) { VkDevice dev = m_pDriver->GetDev(); - VkCmdBuffer cmd = m_pDriver->GetCmd(); - VkQueue q = m_pDriver->GetQ(); + VkCmdBuffer cmd = m_pDriver->GetNextCmd(); const VkLayerDispatchTable *vt = ObjDisp(dev); ImgState &iminfo = m_pDriver->m_ImageInfo[cfg.texid]; @@ -698,8 +696,8 @@ bool VulkanReplay::RenderTextureInternal(TextureDisplay cfg, VkRenderPassBeginIn liveImView = iminfo.view; } - - // VKTODOHIGH once we stop doing DeviceWaitIdle/QueueWaitIdle all over, this + + // VKTODOHIGH once we stop doing QueueWaitIdle after each flip, this // needs to be ring-buffered displayuniforms *data = (displayuniforms *)GetDebugManager()->m_TexDisplayUBO.Map(vt, dev); @@ -838,13 +836,6 @@ bool VulkanReplay::RenderTextureInternal(TextureDisplay cfg, VkRenderPassBeginIn vt->EndCommandBuffer(Unwrap(cmd)); - vt->QueueSubmit(Unwrap(q), 1, UnwrapPtr(cmd), VK_NULL_HANDLE); - - // VKTODOMED ideally all the commands from Bind to Flip would be recorded - // into a single command buffer and we can just have several allocated - // ring-buffer style - vt->QueueWaitIdle(Unwrap(q)); - return true; } @@ -857,8 +848,7 @@ void VulkanReplay::RenderCheckerboard(Vec3f light, Vec3f dark) OutputWindow &outw = it->second; VkDevice dev = m_pDriver->GetDev(); - VkCmdBuffer cmd = m_pDriver->GetCmd(); - VkQueue q = m_pDriver->GetQ(); + VkCmdBuffer cmd = m_pDriver->GetNextCmd(); const VkLayerDispatchTable *vt = ObjDisp(dev); VkCmdBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_CMD_BUFFER_BEGIN_INFO, NULL, VK_CMD_BUFFER_OPTIMIZE_SMALL_BATCH_BIT | VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT }; @@ -868,7 +858,7 @@ void VulkanReplay::RenderCheckerboard(Vec3f light, Vec3f dark) vkr = vt->BeginCommandBuffer(Unwrap(cmd), &beginInfo); RDCASSERT(vkr == VK_SUCCESS); - // VKTODOHIGH once we stop doing DeviceWaitIdle/QueueWaitIdle all over, this + // VKTODOHIGH once we stop doing QueueWaitIdle after each flip, this // needs to be ring-buffered Vec4f *data = (Vec4f *)GetDebugManager()->m_CheckerboardUBO.Map(vt, dev); data[0].x = light.x; @@ -901,14 +891,6 @@ void VulkanReplay::RenderCheckerboard(Vec3f light, Vec3f dark) vkr = vt->EndCommandBuffer(Unwrap(cmd)); RDCASSERT(vkr == VK_SUCCESS); - - vkr = vt->QueueSubmit(Unwrap(q), 1, UnwrapPtr(cmd), VK_NULL_HANDLE); - RDCASSERT(vkr == VK_SUCCESS); - - // VKTODOMED ideally all the commands from Bind to Flip would be recorded - // into a single command buffer and we can just have several allocated - // ring-buffer style - vt->QueueWaitIdle(Unwrap(q)); } void VulkanReplay::RenderHighlightBox(float w, float h, float scale) @@ -920,8 +902,7 @@ void VulkanReplay::RenderHighlightBox(float w, float h, float scale) OutputWindow &outw = it->second; VkDevice dev = m_pDriver->GetDev(); - VkCmdBuffer cmd = m_pDriver->GetCmd(); - VkQueue q = m_pDriver->GetQ(); + VkCmdBuffer cmd = m_pDriver->GetNextCmd(); const VkLayerDispatchTable *vt = ObjDisp(dev); VkCmdBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_CMD_BUFFER_BEGIN_INFO, NULL, VK_CMD_BUFFER_OPTIMIZE_SMALL_BATCH_BIT | VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT }; @@ -937,7 +918,7 @@ void VulkanReplay::RenderHighlightBox(float w, float h, float scale) const float xdim = scale*xpixdim; const float ydim = scale*ypixdim; - // VKTODOHIGH once we stop doing DeviceWaitIdle/QueueWaitIdle all over, this + // VKTODOHIGH once we stop doing QueueWaitIdle after each flip, this // needs to be ring-buffered genericuniforms *data = (genericuniforms *)GetDebugManager()->m_GenericUBO.Map(vt, dev); data->Offset = Vec4f(0.0f, 0.0f, 0.0f, 0.0f); @@ -980,14 +961,6 @@ void VulkanReplay::RenderHighlightBox(float w, float h, float scale) vkr = vt->EndCommandBuffer(Unwrap(cmd)); RDCASSERT(vkr == VK_SUCCESS); - - vkr = vt->QueueSubmit(Unwrap(q), 1, UnwrapPtr(cmd), VK_NULL_HANDLE); - RDCASSERT(vkr == VK_SUCCESS); - - // VKTODOMED ideally all the commands from Bind to Flip would be recorded - // into a single command buffer and we can just have several allocated - // ring-buffer style - vt->QueueWaitIdle(Unwrap(q)); } ResourceId VulkanReplay::RenderOverlay(ResourceId texid, TextureDisplayOverlay overlay, uint32_t frameID, uint32_t eventID, const vector &passEvents) @@ -1047,8 +1020,7 @@ void VulkanReplay::BindOutputWindow(uint64_t id, bool depth) m_DebugHeight = (int32_t)outw.height; VkDevice dev = m_pDriver->GetDev(); - VkCmdBuffer cmd = m_pDriver->GetCmd(); - VkQueue q = m_pDriver->GetQ(); + VkCmdBuffer cmd = m_pDriver->GetNextCmd(); const VkLayerDispatchTable *vt = ObjDisp(dev); // semaphore is short lived, so not wrapped, if it's cached (ideally) @@ -1062,7 +1034,7 @@ void VulkanReplay::BindOutputWindow(uint64_t id, bool depth) vkr = vt->AcquireNextImageKHR(Unwrap(dev), Unwrap(outw.swap), UINT64_MAX, sem, &outw.curidx); RDCASSERT(vkr == VK_SUCCESS); - vkr = vt->QueueWaitSemaphore(Unwrap(q), sem); + vkr = vt->QueueWaitSemaphore(Unwrap(m_pDriver->GetQ()), sem); RDCASSERT(vkr == VK_SUCCESS); vt->DestroySemaphore(Unwrap(dev), sem); @@ -1088,13 +1060,6 @@ void VulkanReplay::BindOutputWindow(uint64_t id, bool depth) outw.coltrans[outw.curidx].oldLayout = outw.bbtrans.newLayout; vt->EndCommandBuffer(Unwrap(cmd)); - - vt->QueueSubmit(Unwrap(q), 1, UnwrapPtr(cmd), VK_NULL_HANDLE); - - // VKTODOMED ideally all the commands from Bind to Flip would be recorded - // into a single command buffer and we can just have several allocated - // ring-buffer style - vt->QueueWaitIdle(Unwrap(q)); } void VulkanReplay::ClearOutputWindowColour(uint64_t id, float col[4]) @@ -1106,8 +1071,7 @@ void VulkanReplay::ClearOutputWindowColour(uint64_t id, float col[4]) OutputWindow &outw = it->second; VkDevice dev = m_pDriver->GetDev(); - VkCmdBuffer cmd = m_pDriver->GetCmd(); - VkQueue q = m_pDriver->GetQ(); + VkCmdBuffer cmd = m_pDriver->GetNextCmd(); const VkLayerDispatchTable *vt = ObjDisp(dev); VkCmdBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_CMD_BUFFER_BEGIN_INFO, NULL, VK_CMD_BUFFER_OPTIMIZE_SMALL_BATCH_BIT | VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT }; @@ -1120,20 +1084,13 @@ void VulkanReplay::ClearOutputWindowColour(uint64_t id, float col[4]) vt->CmdClearColorImage(Unwrap(cmd), Unwrap(outw.bb), VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, (VkClearColorValue *)col, 1, &outw.bbtrans.subresourceRange); vt->EndCommandBuffer(Unwrap(cmd)); - - vt->QueueSubmit(Unwrap(q), 1, UnwrapPtr(cmd), VK_NULL_HANDLE); - - // VKTODOMED ideally all the commands from Bind to Flip would be recorded - // into a single command buffer and we can just have several allocated - // ring-buffer style - vt->QueueWaitIdle(Unwrap(q)); } void VulkanReplay::ClearOutputWindowDepth(uint64_t id, float depth, uint8_t stencil) { VULKANNOTIMP("ClearOutputWindowDepth"); - // VKTODOMED: same as FlipOutputWindow but do a depth clear + // VKTODOMED: same as ClearOutputWindowColour but do a depth clear } void VulkanReplay::FlipOutputWindow(uint64_t id) @@ -1145,8 +1102,7 @@ void VulkanReplay::FlipOutputWindow(uint64_t id) OutputWindow &outw = it->second; VkDevice dev = m_pDriver->GetDev(); - VkCmdBuffer cmd = m_pDriver->GetCmd(); - VkQueue q = m_pDriver->GetQ(); + VkCmdBuffer cmd = m_pDriver->GetNextCmd(); const VkLayerDispatchTable *vt = ObjDisp(dev); VkCmdBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_CMD_BUFFER_BEGIN_INFO, NULL, VK_CMD_BUFFER_OPTIMIZE_SMALL_BATCH_BIT | VK_CMD_BUFFER_OPTIMIZE_ONE_TIME_SUBMIT_BIT }; @@ -1185,15 +1141,14 @@ void VulkanReplay::FlipOutputWindow(uint64_t id) vt->EndCommandBuffer(Unwrap(cmd)); - vt->QueueSubmit(Unwrap(q), 1, UnwrapPtr(cmd), VK_NULL_HANDLE); + // submit all the cmds we recorded + m_pDriver->SubmitCmds(); VkPresentInfoKHR presentInfo = { VK_STRUCTURE_TYPE_PRESENT_INFO_KHR, NULL, 1, UnwrapPtr(outw.swap), &outw.curidx }; - vt->QueuePresentKHR(Unwrap(q), &presentInfo); + vt->QueuePresentKHR(Unwrap(m_pDriver->GetQ()), &presentInfo); - vt->QueueWaitIdle(Unwrap(q)); - - vt->DeviceWaitIdle(Unwrap(dev)); + m_pDriver->FlushQ(); } void VulkanReplay::DestroyOutputWindow(uint64_t id) @@ -1233,8 +1188,7 @@ uint64_t VulkanReplay::MakeOutputWindow(void *wn, bool depth) vector VulkanReplay::GetBufferData(ResourceId buff, uint32_t offset, uint32_t len) { VkDevice dev = m_pDriver->GetDev(); - VkCmdBuffer cmd = m_pDriver->GetCmd(); - VkQueue q = m_pDriver->GetQ(); + VkCmdBuffer cmd = m_pDriver->GetNextCmd(); const VkLayerDispatchTable *vt = ObjDisp(dev); ResourceId memid; @@ -1311,14 +1265,11 @@ vector VulkanReplay::GetBufferData(ResourceId buff, uint32_t offset, uint3 vkr = vt->EndCommandBuffer(Unwrap(cmd)); RDCASSERT(vkr == VK_SUCCESS); - - vkr = vt->QueueSubmit(Unwrap(q), 1, UnwrapPtr(cmd), VK_NULL_HANDLE); - RDCASSERT(vkr == VK_SUCCESS); - - vkr = vt->QueueWaitIdle(Unwrap(q)); - RDCASSERT(vkr == VK_SUCCESS); } + m_pDriver->SubmitCmds(); + m_pDriver->FlushQ(); + vkr = vt->MapMemory(Unwrap(dev), readbackmem, 0, 0, 0, (void **)&pData); RDCASSERT(vkr == VK_SUCCESS); diff --git a/renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp b/renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp index 46122d500..19a09662a 100644 --- a/renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp +++ b/renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp @@ -307,12 +307,6 @@ bool WrappedVulkan::Serialise_vkCreateDevice( GetResourceManager()->WrapResource(Unwrap(device), m_PhysicalReplayData[i].cmdpool); - VkCmdBufferCreateInfo cmdInfo = { VK_STRUCTURE_TYPE_CMD_BUFFER_CREATE_INFO, NULL, Unwrap(m_PhysicalReplayData[i].cmdpool), VK_CMD_BUFFER_LEVEL_PRIMARY, 0 }; - vkr = ObjDisp(device)->CreateCommandBuffer(Unwrap(device), &cmdInfo, &m_PhysicalReplayData[i].cmd); - RDCASSERT(vkr == VK_SUCCESS); - - GetResourceManager()->WrapResource(Unwrap(device), m_PhysicalReplayData[i].cmd); - found = true; break; } @@ -450,18 +444,13 @@ VkResult WrappedVulkan::vkCreateDevice( GetResourceManager()->WrapResource(Unwrap(*pDevice), m_PhysicalReplayData[i].cmdpool); - VkCmdBufferCreateInfo cmdInfo = { VK_STRUCTURE_TYPE_CMD_BUFFER_CREATE_INFO, NULL, Unwrap(m_PhysicalReplayData[i].cmdpool), VK_CMD_BUFFER_LEVEL_PRIMARY, 0 }; - vkr = ObjDisp(*pDevice)->CreateCommandBuffer(Unwrap(*pDevice), &cmdInfo, &m_PhysicalReplayData[i].cmd); - RDCASSERT(vkr == VK_SUCCESS); - found = true; - - GetResourceManager()->WrapResource(Unwrap(*pDevice), m_PhysicalReplayData[i].cmd); - // VKTODOHIGH hack, need to properly handle multiple devices etc and // not have this 'current swap chain device' thing. m_SwapPhysDevice = (int)i; m_PhysicalReplayData[i].debugMan = new VulkanDebugManager(this, *pDevice); + + found = true; break; } } @@ -484,9 +473,6 @@ void WrappedVulkan::vkDestroyDevice(VkDevice device) { if(m_PhysicalReplayData[i].dev == device) { - if(m_PhysicalReplayData[i].cmd != VK_NULL_HANDLE) - ObjDisp(device)->DestroyCommandBuffer(Unwrap(device), Unwrap(m_PhysicalReplayData[i].cmd)); - if(m_PhysicalReplayData[i].cmdpool != VK_NULL_HANDLE) ObjDisp(device)->DestroyCommandPool(Unwrap(device), Unwrap(m_PhysicalReplayData[i].cmdpool)); diff --git a/renderdoc/driver/vulkan/wrappers/vk_wsi_funcs.cpp b/renderdoc/driver/vulkan/wrappers/vk_wsi_funcs.cpp index 4ac4f2b48..10cfa2db9 100644 --- a/renderdoc/driver/vulkan/wrappers/vk_wsi_funcs.cpp +++ b/renderdoc/driver/vulkan/wrappers/vk_wsi_funcs.cpp @@ -525,15 +525,9 @@ VkResult WrappedVulkan::vkQueuePresentKHR( VkRenderPass rp = swapInfo.rp; VkFramebuffer fb = swapInfo.images[pPresentInfo->imageIndices[0]].fb; - // VKTODOLOW only handling queue == GetQ() - RDCASSERT(GetQ() == queue); - VkQueue q = GetQ(); - VkLayerDispatchTable *vt = ObjDisp(GetDev()); - vt->QueueWaitIdle(Unwrap(q)); - - TextPrintState textstate = { q, GetCmd(), rp, fb, swapInfo.extent.width, swapInfo.extent.height }; + TextPrintState textstate = { VK_NULL_HANDLE, rp, fb, swapInfo.extent.width, swapInfo.extent.height }; if(activeWindow) { @@ -566,13 +560,19 @@ VkResult WrappedVulkan::vkQueuePresentKHR( if(!overlayText.empty()) { + textstate.cmd = GetNextCmd(); GetDebugManager()->RenderText(textstate, 0.0f, y, overlayText.c_str()); + SubmitCmds(); + FlushQ(); y += 1.0f; } if(overlay & eRENDERDOC_Overlay_CaptureList) { + textstate.cmd = GetNextCmd(); GetDebugManager()->RenderText(textstate, 0.0f, y, "%d Captures saved.\n", (uint32_t)m_FrameRecord.size()); + SubmitCmds(); + FlushQ(); y += 1.0f; uint64_t now = Timing::GetUnixTimestamp(); @@ -580,7 +580,10 @@ VkResult WrappedVulkan::vkQueuePresentKHR( { if(now - m_FrameRecord[i].frameInfo.captureTime < 20) { + textstate.cmd = GetNextCmd(); GetDebugManager()->RenderText(textstate, 0.0f, y, "Captured frame %d.\n", m_FrameRecord[i].frameInfo.frameNumber); + SubmitCmds(); + FlushQ(); y += 1.0f; } } @@ -589,7 +592,10 @@ VkResult WrappedVulkan::vkQueuePresentKHR( // VKTODOLOW failed frames #if !defined(RELEASE) + textstate.cmd = GetNextCmd(); GetDebugManager()->RenderText(textstate, 0.0f, y, "%llu chunks - %.2f MB", Chunk::NumLiveChunks(), float(Chunk::TotalMem())/1024.0f/1024.0f); + SubmitCmds(); + FlushQ(); y += 1.0f; #endif } @@ -611,9 +617,16 @@ VkResult WrappedVulkan::vkQueuePresentKHR( if(!keys.empty()) str += " to cycle between swapchains"; - + + textstate.cmd = GetNextCmd(); GetDebugManager()->RenderText(textstate, 0.0f, 0.0f, str.c_str()); + SubmitCmds(); + FlushQ(); } + + // VKTODOLOW once rendertext can be called multiple times (with e.g. dynamic UBO) + // submit cmds here and don't flush + //SubmitCmds(); } } @@ -644,8 +657,7 @@ VkResult WrappedVulkan::vkQueuePresentKHR( if(1)//if(wnd) { VkDevice dev = GetDev(); - VkQueue q = GetQ(); - VkCmdBuffer cmd = GetCmd(); + VkCmdBuffer cmd = GetNextCmd(); const VkLayerDispatchTable *vt = ObjDisp(dev); @@ -743,11 +755,8 @@ VkResult WrappedVulkan::vkQueuePresentKHR( vkr = vt->EndCommandBuffer(Unwrap(cmd)); RDCASSERT(vkr == VK_SUCCESS); - vkr = vt->QueueSubmit(Unwrap(q), 1, UnwrapPtr(cmd), VK_NULL_HANDLE); - RDCASSERT(vkr == VK_SUCCESS); - - // wait queue idle - vt->QueueWaitIdle(Unwrap(q)); + SubmitCmds(); + FlushQ(); // need to wait so we can readback // map memory and readback byte *pData = NULL;