From a68c0763413a90fb03f8392c7a4b0a67c70cbf7c Mon Sep 17 00:00:00 2001 From: Cam Mannett Date: Thu, 3 Oct 2024 15:51:51 +0100 Subject: [PATCH] Switch to GPU side query result replacement --- .../vulkan/vk_acceleration_structure.cpp | 18 ++-- .../driver/vulkan/vk_acceleration_structure.h | 7 +- renderdoc/driver/vulkan/vk_resources.cpp | 102 ++---------------- renderdoc/driver/vulkan/vk_resources.h | 34 +----- .../driver/vulkan/wrappers/vk_cmd_funcs.cpp | 83 +++++++++----- .../driver/vulkan/wrappers/vk_misc_funcs.cpp | 48 ++++++++- 6 files changed, 125 insertions(+), 167 deletions(-) diff --git a/renderdoc/driver/vulkan/vk_acceleration_structure.cpp b/renderdoc/driver/vulkan/vk_acceleration_structure.cpp index c57120297..56fe148db 100644 --- a/renderdoc/driver/vulkan/vk_acceleration_structure.cpp +++ b/renderdoc/driver/vulkan/vk_acceleration_structure.cpp @@ -75,7 +75,7 @@ VulkanAccelerationStructureManager::VulkanAccelerationStructureManager(WrappedVu RDResult VulkanAccelerationStructureManager::CopyInputBuffers( VkCommandBuffer commandBuffer, const VkAccelerationStructureBuildGeometryInfoKHR &info, - const VkAccelerationStructureBuildRangeInfoKHR *buildRange, CaptureState state) + const VkAccelerationStructureBuildRangeInfoKHR *buildRange) { VkResourceRecord *cmdRecord = GetRecord(commandBuffer); RDCASSERT(cmdRecord); @@ -87,7 +87,7 @@ RDResult VulkanAccelerationStructureManager::CopyInputBuffers( VkAccelerationStructureInfo *metadata = asRecord->accelerationStructureInfo; if(!metadata->geometryData.empty()) { - DeletePreviousInfo(commandBuffer, metadata, state); + DeletePreviousInfo(commandBuffer, metadata); metadata = asRecord->accelerationStructureInfo = new VkAccelerationStructureInfo(); } @@ -360,8 +360,8 @@ RDResult VulkanAccelerationStructureManager::CopyInputBuffers( geometry.geometry.instances; if(instanceInfo.arrayOfPointers) - return RDResult(ResultCode::InternalError, - "AS instance build arrayOfPointers unsupported"); + RETURN_ERROR_RESULT(ResultCode::InternalError, + "AS instance build arrayOfPointers unsupported"); // Find the associated VkBuffer BufferData data = GetDeviceAddressData(instanceInfo.data.deviceAddress); @@ -432,7 +432,7 @@ RDResult VulkanAccelerationStructureManager::CopyInputBuffers( } void VulkanAccelerationStructureManager::CopyAccelerationStructure( - VkCommandBuffer commandBuffer, const VkCopyAccelerationStructureInfoKHR &pInfo, CaptureState state) + VkCommandBuffer commandBuffer, const VkCopyAccelerationStructureInfoKHR &pInfo) { VkResourceRecord *srcRecord = GetRecord(pInfo.src); RDCASSERT(srcRecord->accelerationStructureInfo != NULL); @@ -441,7 +441,7 @@ void VulkanAccelerationStructureManager::CopyAccelerationStructure( VkResourceRecord *dstRecord = GetRecord(pInfo.dst); VkAccelerationStructureInfo *info = dstRecord->accelerationStructureInfo; if(!info->geometryData.empty()) - DeletePreviousInfo(commandBuffer, info, state); + DeletePreviousInfo(commandBuffer, info); // Rather than copy the backing mem, we can just increase the ref count. If there is an update // build to the AS then the ref will be replaced in the record, so there's no risk of aliasing. @@ -859,8 +859,7 @@ VulkanAccelerationStructureManager::RecordAndOffset VulkanAccelerationStructureM } template -void VulkanAccelerationStructureManager::DeletePreviousInfo(VkCommandBuffer commandBuffer, T *info, - CaptureState state) +void VulkanAccelerationStructureManager::DeletePreviousInfo(VkCommandBuffer commandBuffer, T *info) { VkResourceRecord *cmdRecord = GetRecord(commandBuffer); cmdRecord->cmdInfo->pendingSubmissionCompleteCallbacks->callbacks.push_back( @@ -869,8 +868,7 @@ void VulkanAccelerationStructureManager::DeletePreviousInfo(VkCommandBuffer comm // OMM suport todo template void VulkanAccelerationStructureManager::DeletePreviousInfo(VkCommandBuffer commandBuffer, - VkAccelerationStructureInfo *info, - CaptureState state); + VkAccelerationStructureInfo *info); VkDeviceSize VulkanAccelerationStructureManager::SerialisedASSize(VkAccelerationStructureKHR unwrappedAs) { diff --git a/renderdoc/driver/vulkan/vk_acceleration_structure.h b/renderdoc/driver/vulkan/vk_acceleration_structure.h index 74aed5d65..7eae153c6 100644 --- a/renderdoc/driver/vulkan/vk_acceleration_structure.h +++ b/renderdoc/driver/vulkan/vk_acceleration_structure.h @@ -107,14 +107,13 @@ public: // are copied. RDResult CopyInputBuffers(VkCommandBuffer commandBuffer, const VkAccelerationStructureBuildGeometryInfoKHR &info, - const VkAccelerationStructureBuildRangeInfoKHR *buildRange, - CaptureState state); + const VkAccelerationStructureBuildRangeInfoKHR *buildRange); // Copies the metadata from src to dst, the input buffers are identical so don't need to be // duplicated. Compaction is ignored but the copy is still performed so the dst handle is valid // on replay void CopyAccelerationStructure(VkCommandBuffer commandBuffer, - const VkCopyAccelerationStructureInfoKHR &pInfo, CaptureState state); + const VkCopyAccelerationStructureInfoKHR &pInfo); // Called when the initial state is prepared. Any TLAS and BLAS data is copied into temporary // buffers and the handles for that memory and the buffers is stored in the init state @@ -135,7 +134,7 @@ private: RecordAndOffset GetDeviceAddressData(VkDeviceAddress address) const; template - void DeletePreviousInfo(VkCommandBuffer commandBuffer, T *info, CaptureState state); + void DeletePreviousInfo(VkCommandBuffer commandBuffer, T *info); VkDeviceSize SerialisedASSize(VkAccelerationStructureKHR unwrappedAs); diff --git a/renderdoc/driver/vulkan/vk_resources.cpp b/renderdoc/driver/vulkan/vk_resources.cpp index 4c5e0fc62..b73a8fcd3 100644 --- a/renderdoc/driver/vulkan/vk_resources.cpp +++ b/renderdoc/driver/vulkan/vk_resources.cpp @@ -3948,105 +3948,17 @@ void ImgRefs::Split(bool splitAspects, bool splitLevels, bool splitLayers) areLayersSplit = newSplitLayerCount > 1; } -void QueryPoolInfo::Add(uint32_t firstQuery, rdcarray values) +QueryPoolInfo::QueryPoolInfo(WrappedVulkan *driver, VkDevice device, + const VkQueryPoolCreateInfo *pCreateInfo) { - Reset(firstQuery, (uint32_t)values.size()); - - m_Entries.reserve(m_Entries.size() + values.size()); - for(uint64_t value : values) - m_Entries.emplace_back(firstQuery++, value); - - std::sort(m_Entries.begin(), m_Entries.end()); + m_Buffer.Create(driver, device, pCreateInfo->queryCount * 8, 1, GPUBuffer::eGPUBufferReadback); + m_MappedMem = (byte *)m_Buffer.Map(0, m_Buffer.totalsize); } -void QueryPoolInfo::Reset(uint32_t firstQuery, uint32_t queryCount) +QueryPoolInfo::~QueryPoolInfo() { - m_Entries.removeIf([&](const auto &entry) { - return (entry.index >= firstQuery) && (entry.index < (firstQuery + queryCount)); - }); -} - -void QueryPoolInfo::Replace(uint32_t firstQuery, uint32_t queryCount, void *pData, - VkDeviceSize stride, VkQueryResultFlags flags) const -{ - const auto writeEntry = [&](Entry queryPoolInfoEntry) { - const size_t num_bytes = (flags & VK_QUERY_RESULT_64_BIT) ? 8 : 4; - - byte *pStart = (byte *)pData + (queryPoolInfoEntry.index * stride); - memcpy(pStart, &queryPoolInfoEntry.value, num_bytes); - }; - - Replace(firstQuery, queryCount, writeEntry); -} - -void QueryPoolInfo::Replace(uint32_t firstQuery, uint32_t queryCount, - const std::function)> &writeEntry) const -{ - rdcarray entries; - entries.reserve(queryCount); - - // Swap out any AS compaction sizes with the replacements - Replace(firstQuery, queryCount, [&](Entry entry) { entries.push_back(entry); }); - - std::sort(entries.begin(), entries.end()); - - // Now batch into contiguous ranges and dispatch - for(size_t i = 0; i < entries.size();) - { - uint32_t queryIndex = entries[i].index; - rdcarray batch; - - while(queryIndex == entries[++i].index) - { - batch.push_back(entries[i].value); - ++queryIndex; - } - - writeEntry(queryIndex, std::move(batch)); - } -} - -bool QueryPoolInfo::HasReplacementEntries(uint32_t firstQuery, uint32_t queryCount) const -{ - uint32_t start, end; - rdctie(start, end) = GetIntersection(firstQuery, queryCount); - return start <= end; -} - -rdcpair QueryPoolInfo::GetIntersection(uint32_t firstQuery, - uint32_t queryCount) const -{ - if(m_Entries.empty()) - return {1, 0}; // Invalid - - const uint32_t start = RDCMAX(firstQuery, m_Entries.front().index); - const uint32_t end = RDCMIN(firstQuery + queryCount - 1, (uint32_t)m_Entries.back().index); - return {start, end}; -} - -void QueryPoolInfo::Replace(uint32_t firstQuery, uint32_t queryCount, - const std::function &writeEntry) const -{ - if(!m_Entries.empty()) - { - // Find the intersection of the two query ranges - uint32_t start, end; - rdctie(start, end) = GetIntersection(firstQuery, queryCount); - if(end < start) - return; - - uint32_t j = 0; - for(uint32_t i = start; i < end; ++i) - { - // The indices are sparse but ordered - while(i != m_Entries[j].index) - { - ++j; - } - - writeEntry(m_Entries[j]); - } - } + m_Buffer.Unmap(); + m_Buffer.Destroy(); } VkResourceRecord::~VkResourceRecord() diff --git a/renderdoc/driver/vulkan/vk_resources.h b/renderdoc/driver/vulkan/vk_resources.h index ad6f25e90..5dbdab073 100644 --- a/renderdoc/driver/vulkan/vk_resources.h +++ b/renderdoc/driver/vulkan/vk_resources.h @@ -2236,39 +2236,15 @@ inline FrameRefType MarkMemoryReferenced(std::unordered_map return MarkMemoryReferenced(memRefs, mem, offset, size, refType, ComposeFrameRefs); } -// Used to replace QueryPool results +// Used as an alternative backing store to VkQueryPool to replace query results class QueryPoolInfo { public: - void Add(uint32_t firstQuery, rdcarray values); + QueryPoolInfo(WrappedVulkan *driver, VkDevice device, const VkQueryPoolCreateInfo *pCreateInfo); + ~QueryPoolInfo(); - void Reset(uint32_t firstQuery, uint32_t queryCount); - - void Replace(uint32_t firstQuery, uint32_t queryCount, void *pData, VkDeviceSize stride, - VkQueryResultFlags flags) const; - - // Calls writeEntry with matching contiguous entries, buffered into an array. - void Replace(uint32_t firstQuery, uint32_t queryCount, - const std::function)> &writeEntry) const; - - bool HasReplacementEntries(uint32_t firstQuery, uint32_t queryCount) const; - -private: - struct Entry - { - Entry(uint32_t i, uint64_t v) : index(i), value(v) {} - bool operator<(Entry other) const { return index < other.index; } - - uint32_t index; - uint64_t value; - }; - - rdcpair GetIntersection(uint32_t firstQuery, uint32_t queryCount) const; - - void Replace(uint32_t firstQuery, uint32_t queryCount, - const std::function &writeEntry) const; - - rdcarray m_Entries; + GPUBuffer m_Buffer; + byte *m_MappedMem; }; struct DescUpdateTemplate; diff --git a/renderdoc/driver/vulkan/wrappers/vk_cmd_funcs.cpp b/renderdoc/driver/vulkan/wrappers/vk_cmd_funcs.cpp index ca4127c20..bee67e097 100644 --- a/renderdoc/driver/vulkan/wrappers/vk_cmd_funcs.cpp +++ b/renderdoc/driver/vulkan/wrappers/vk_cmd_funcs.cpp @@ -4539,8 +4539,9 @@ void WrappedVulkan::vkCmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, VkQ record->MarkResourceFrameReferenced(GetResID(queryPool), eFrameRef_Read); + const bool is64bit = (flags & VK_QUERY_RESULT_64_BIT) > 0; VkDeviceSize size = (queryCount - 1) * destStride + 4; - if(flags & VK_QUERY_RESULT_64_BIT) + if(is64bit) { size += 4; } @@ -4548,34 +4549,57 @@ void WrappedVulkan::vkCmdCopyQueryPoolResults(VkCommandBuffer commandBuffer, VkQ eFrameRef_PartialWrite); const QueryPoolInfo *qpInfo = GetRecord(queryPool)->queryPoolInfo; - if(qpInfo->HasReplacementEntries(firstQuery, queryCount)) + if(qpInfo) { - // We want to record these commands into the capture so they are replayed VkMemoryBarrier barrier = { VK_STRUCTURE_TYPE_MEMORY_BARRIER, NULL, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT, }; - vkCmdPipelineBarrier(commandBuffer, VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0, 1, &barrier, 0, VK_NULL_HANDLE, 0, - VK_NULL_HANDLE); + ObjDisp(commandBuffer) + ->CmdPipelineBarrier(Unwrap(commandBuffer), VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0, 1, &barrier, 0, VK_NULL_HANDLE, + 0, VK_NULL_HANDLE); - qpInfo->Replace( - firstQuery, queryCount, [&](uint32_t queryIndexStart, rdcarray results) { - const size_t size = (size_t)(results.size() * destStride); - RDCASSERT(size < (1 << 16)); + const bool hasAvailability = (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) > 0; + const VkDeviceSize resultSize = is64bit ? sizeof(uint64_t) : sizeof(uint32_t); - const size_t resultSize = (flags & VK_QUERY_RESULT_64_BIT) ? 8 : 4; - byte *tmp = new byte[size]; - size_t i = 0; - for(byte *ptr = tmp; ptr < (ptr + size); ptr += destStride) - memcpy(ptr, &results[i++], resultSize); + // If the stride matches the source buffer, then we can copy everything in a single region + if(is64bit && !hasAvailability && destStride == resultSize) + { + const VkBufferCopy region = { + firstQuery * resultSize, + destOffset, + queryCount * resultSize, + }; + ObjDisp(commandBuffer) + ->CmdCopyBuffer(Unwrap(commandBuffer), Unwrap(qpInfo->m_Buffer.buf), Unwrap(destBuffer), + 1, ®ion); + } + else + { + // Copy each result into the destination, converting if required + rdcarray regions; + regions.reserve(queryCount); + for(size_t i = 0; i < queryCount; ++i) + regions.push_back( + {(firstQuery + i) * sizeof(uint64_t), destOffset + (i * destStride), resultSize}); - vkCmdUpdateBuffer(commandBuffer, destBuffer, destOffset + (queryIndexStart * destStride), - size, (const uint32_t *)tmp); - delete[] tmp; - }); + ObjDisp(commandBuffer) + ->CmdCopyBuffer(Unwrap(commandBuffer), Unwrap(qpInfo->m_Buffer.buf), Unwrap(destBuffer), + (uint32_t)regions.size(), regions.data()); + } + + if(hasAvailability) + { + const uint64_t availability = 1; + for(size_t i = 0; i < queryCount; ++i) + ObjDisp(commandBuffer) + ->CmdUpdateBuffer(Unwrap(commandBuffer), Unwrap(qpInfo->m_Buffer.buf), + destOffset + (queryCount * resultSize) + resultSize, resultSize, + (uint32_t *)&availability); + } } } } @@ -7906,11 +7930,10 @@ void WrappedVulkan::vkCmdBuildAccelerationStructuresKHR( record->cmdInfo->accelerationStructures.push_back(GetRecord(geomInfo.dstAccelerationStructure)); const RDResult copyResult = GetAccelerationStructureManager()->CopyInputBuffers( - commandBuffer, geomInfo, ppBuildRangeInfos[i], m_State); + commandBuffer, geomInfo, ppBuildRangeInfos[i]); if(copyResult != ResultCode::Succeeded) { m_LastCaptureError = copyResult; - RDCERR("%s", copyResult.message.c_str()); m_CaptureFailure = true; } } @@ -7964,6 +7987,8 @@ void WrappedVulkan::vkCmdCopyAccelerationStructureKHR(VkCommandBuffer commandBuf // Add to the command buffer metadata, so we can know when it has been submitted record->cmdInfo->accelerationStructures.push_back(GetRecord(pInfo->dst)); + + GetAccelerationStructureManager()->CopyAccelerationStructure(commandBuffer, *pInfo); } } @@ -8051,16 +8076,24 @@ void WrappedVulkan::vkCmdWriteAccelerationStructuresPropertiesKHR( // The compacted size can vary between capture and replay, so to ensure we always have enough // memory we return the full AS size - if(queryType == VK_QUERY_TYPE_ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR) + QueryPoolInfo *qpInfo = GetRecord(queryPool)->queryPoolInfo; + if(qpInfo) { - auto &qpInfo = GetRecord(queryPool)->queryPoolInfo; - rdcarray sizes; sizes.reserve(accelerationStructureCount); for(uint32_t i = 0; i < accelerationStructureCount; ++i) sizes.push_back(GetRecord(pAccelerationStructures[i])->memSize); - qpInfo->Add(firstQuery, std::move(sizes)); + constexpr size_t maxTransferrableBytes = 65536; + const size_t totalBytes = sizes.size() * sizeof(uint64_t); + for(size_t i = 0; i < totalBytes; i += maxTransferrableBytes) + { + const VkDeviceSize numBytes = RDCMIN(maxTransferrableBytes, totalBytes); + const VkDeviceSize startOffset = (firstQuery * sizeof(uint64_t)) + i; + ObjDisp(commandBuffer) + ->CmdUpdateBuffer(Unwrap(commandBuffer), Unwrap(qpInfo->m_Buffer.buf), startOffset, + numBytes, (uint32_t *)((byte *)sizes.data() + i)); + } } ObjDisp(commandBuffer) diff --git a/renderdoc/driver/vulkan/wrappers/vk_misc_funcs.cpp b/renderdoc/driver/vulkan/wrappers/vk_misc_funcs.cpp index ab374e11b..705f60a34 100644 --- a/renderdoc/driver/vulkan/wrappers/vk_misc_funcs.cpp +++ b/renderdoc/driver/vulkan/wrappers/vk_misc_funcs.cpp @@ -1649,7 +1649,15 @@ VkResult WrappedVulkan::vkCreateQueryPool(VkDevice device, const VkQueryPoolCrea } VkResourceRecord *record = GetResourceManager()->AddResourceRecord(*pQueryPool); - record->queryPoolInfo = new QueryPoolInfo(); + + // We swap out the queried compacted AS size for the uncompacted size as they can differ + // between capture and replay + if(pCreateInfo->queryType == VK_QUERY_TYPE_ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR) + { + record->queryPoolInfo = new QueryPoolInfo(this, device, pCreateInfo); + GetResourceManager()->SetInternalResource(GetResID(record->queryPoolInfo->m_Buffer.buf)); + GetResourceManager()->SetInternalResource(GetResID(record->queryPoolInfo->m_Buffer.mem)); + } record->AddChunk(chunk); } @@ -1670,7 +1678,41 @@ VkResult WrappedVulkan::vkGetQueryPoolResults(VkDevice device, VkQueryPool query VkResult result = ObjDisp(device)->GetQueryPoolResults( Unwrap(device), Unwrap(queryPool), firstQuery, queryCount, dataSize, pData, stride, flags); - GetRecord(queryPool)->queryPoolInfo->Replace(firstQuery, queryCount, pData, stride, flags); + const QueryPoolInfo *qpInfo = GetRecord(queryPool)->queryPoolInfo; + if(qpInfo) + { + VkMappedMemoryRange range = { + VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE, + NULL, + qpInfo->m_Buffer.mem, + firstQuery * sizeof(uint64_t), + queryCount * sizeof(uint64_t), + }; + vkInvalidateMappedMemoryRanges(device, 1, &range); + + const bool is64bit = (flags & VK_QUERY_RESULT_64_BIT) > 0; + const bool hasAvailability = (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) > 0; + const size_t resultSize = is64bit ? sizeof(uint64_t) : sizeof(uint32_t); + + // If the stride matches the source buffer, then we can copy everything in a single block + if(is64bit && !hasAvailability && stride == resultSize) + { + memcpy(pData, qpInfo->m_MappedMem + (firstQuery * resultSize), queryCount * resultSize); + } + else + { + for(size_t i = 0; i < queryCount; ++i) + memcpy((byte *)pData + (i * stride), qpInfo->m_MappedMem + ((firstQuery + i) * resultSize), + resultSize); + } + + if(hasAvailability) + { + const uint64_t availability = 1; + for(size_t i = 0; i < queryCount; ++i) + memcpy((byte *)pData + (queryCount * stride) + resultSize, &availability, resultSize); + } + } return result; } @@ -1705,8 +1747,6 @@ void WrappedVulkan::vkResetQueryPool(VkDevice device, VkQueryPool queryPool, uin SERIALISE_TIME_CALL( ObjDisp(device)->ResetQueryPool(Unwrap(device), Unwrap(queryPool), firstQuery, queryCount)); - GetRecord(queryPool)->queryPoolInfo->Reset(firstQuery, queryCount); - if(IsActiveCapturing(m_State)) { CACHE_THREAD_SERIALISER();