From 8ff59fea795e918611bdccd1cfd1e719d9ffb66f Mon Sep 17 00:00:00 2001 From: baldurk Date: Mon, 8 Mar 2021 14:54:44 +0000 Subject: [PATCH] Snapshot sparse tables into resource initial states, and apply on replay --- renderdoc/driver/vulkan/vk_initstate.cpp | 459 ++++++++++++++++++++++- renderdoc/driver/vulkan/vk_manager.h | 47 ++- 2 files changed, 487 insertions(+), 19 deletions(-) diff --git a/renderdoc/driver/vulkan/vk_initstate.cpp b/renderdoc/driver/vulkan/vk_initstate.cpp index 9c603dcaf..73a950893 100644 --- a/renderdoc/driver/vulkan/vk_initstate.cpp +++ b/renderdoc/driver/vulkan/vk_initstate.cpp @@ -36,6 +36,13 @@ // command buffer that stalls the GPU). // See INITSTATEBATCH +template +void DoSerialise(SerialiserType &ser, AspectSparseTable &el) +{ + SERIALISE_MEMBER(aspectMask); + SERIALISE_MEMBER(table); +} + bool WrappedVulkan::Prepare_InitialState(WrappedVkRes *res) { ResourceId id = GetResourceManager()->GetID(res); @@ -72,6 +79,11 @@ bool WrappedVulkan::Prepare_InitialState(WrappedVkRes *res) // buffers are only dirty if they are sparse RDCASSERT(buffer->record->resInfo && buffer->record->resInfo->IsSparse()); + VkInitialContents initialContents(type, VkInitialContents::SparseTableOnly); + + initialContents.SnapshotPageTable(*buffer->record->resInfo); + + GetResourceManager()->SetInitialContents(id, initialContents); return true; } else if(type == eResImage) @@ -82,11 +94,6 @@ bool WrappedVulkan::Prepare_InitialState(WrappedVkRes *res) const ResourceInfo &resInfo = *im->record->resInfo; const ImageInfo &imageInfo = resInfo.imageInfo; - if(resInfo.IsSparse()) - { - // if the image is sparse we have to snapshot the page table - } - LockedImageStateRef state = FindImageState(im->id); // if the image has no memory bound, nothing is to be fetched @@ -415,7 +422,15 @@ bool WrappedVulkan::Prepare_InitialState(WrappedVkRes *res) GetResourceManager()->ReleaseWrappedResource(arrayIm); } - GetResourceManager()->SetInitialContents(id, VkInitialContents(type, readbackmem)); + VkInitialContents initialContents(type, readbackmem); + + // include the sparse page table if it exists + if(resInfo.IsSparse()) + { + initialContents.SnapshotPageTable(resInfo); + } + + GetResourceManager()->SetInitialContents(id, initialContents); return true; } @@ -526,19 +541,35 @@ bool WrappedVulkan::Prepare_InitialState(WrappedVkRes *res) uint64_t WrappedVulkan::GetSize_InitialState(ResourceId id, const VkInitialContents &initial) { + uint64_t ret = 0; + + // account for sparse page tables when present + if(initial.sparseTables) + { + // array count overheads + ret += 128; + for(size_t i = 0; i < initial.sparseTables->size(); i++) + { + ret += sizeof(VkImageAspectFlagBits); + ret += initial.sparseTables->at(i).table.GetSerialiseSize(); + } + } + if(initial.type == eResDescriptorSet) { + // shouldn't have a sparse table here! + RDCASSERTEQUAL(ret, 0); return 128 + initial.numDescriptors * sizeof(DescriptorSetSlot) + initial.inlineByteSize; } else if(initial.type == eResBuffer) { // buffers only have initial states when they're sparse - return 0; + return ret; } else if(initial.type == eResImage || initial.type == eResDeviceMemory) { // the size primarily comes from the buffer, the size of which we conveniently have stored. - return uint64_t(128 + initial.mem.size + WriteSerialiser::GetChunkAlignment()); + return ret + uint64_t(128 + initial.mem.size + WriteSerialiser::GetChunkAlignment()); } RDCERR("Unhandled resource type %s", ToStr(initial.type).c_str()); @@ -558,6 +589,341 @@ static rdcliteral NameOfType(VkResourceType type) return "VkResource"_lit; } +SparseBinding::SparseBinding(WrappedVulkan *vk, VkBuffer unwrappedBuffer, + const rdcarray &tables) +{ + sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO; + pNext = NULL; + waitSemaphoreCount = 0; + pWaitSemaphores = NULL; + imageOpaqueBindCount = 0; + pImageOpaqueBinds = NULL; + imageBindCount = 0; + pImageBinds = NULL; + signalSemaphoreCount = 0; + pSignalSemaphores = NULL; + + if(tables.empty()) + { + RDCERR("Expected a page table initialising buffer sparse bindings"); + invalid = true; + return; + } + + VkDevice device = vk->GetDev(); + + VkMemoryRequirements mrq = {0}; + ObjDisp(device)->GetBufferMemoryRequirements(Unwrap(device), unwrappedBuffer, &mrq); + + const Sparse::PageTable &table = tables[0].table; + + if(mrq.alignment != table.getPageByteSize()) + { + RDCERR("Captured page table uses page size %llu, but on replay page size is %llu", + table.getPageByteSize(), mrq.alignment); + invalid = true; + return; + } + + bufBind.buffer = unwrappedBuffer; + + RDCASSERTEQUAL(table.getMipTail().mappings.size(), 1); + + const Sparse::PageRangeMapping &mapping = table.getMipTail().mappings[0]; + + if(mapping.hasSingleMapping()) + { + opaqueBinds.resize(1); + opaqueBinds[0].flags = 0; + opaqueBinds[0].resourceOffset = 0; + opaqueBinds[0].memory = + Unwrap(vk->GetResourceManager()->GetLiveHandle(mapping.singleMapping.memory)); + opaqueBinds[0].memoryOffset = mapping.singleMapping.offset; + opaqueBinds[0].size = table.getMipTail().totalPackedByteSize; + } + else + { + opaqueBinds.resize(mapping.pages.size()); + for(size_t i = 0; i < mapping.pages.size(); i++) + { + opaqueBinds[i].flags = 0; + opaqueBinds[i].resourceOffset = i * mrq.alignment; + opaqueBinds[i].memory = + Unwrap(vk->GetResourceManager()->GetLiveHandle(mapping.pages[i].memory)); + opaqueBinds[i].memoryOffset = mapping.pages[i].offset; + opaqueBinds[i].size = mrq.alignment; + } + } + + bufBind.bindCount = (uint32_t)opaqueBinds.size(); + bufBind.pBinds = opaqueBinds.data(); + + bufferBindCount = 1; + pBufferBinds = &bufBind; +} + +SparseBinding::SparseBinding(WrappedVulkan *vk, VkImage unwrappedImage, + const rdcarray &tables) +{ + sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO; + pNext = NULL; + waitSemaphoreCount = 0; + pWaitSemaphores = NULL; + bufferBindCount = 0; + pBufferBinds = NULL; + signalSemaphoreCount = 0; + pSignalSemaphores = NULL; + + VkDevice device = vk->GetDev(); + + VkMemoryRequirements mrq = {0}; + ObjDisp(device)->GetImageMemoryRequirements(Unwrap(device), unwrappedImage, &mrq); + + uint32_t numreqs = 8; + VkSparseImageMemoryRequirements reqs[8]; + ObjDisp(device)->GetImageSparseMemoryRequirements(Unwrap(device), unwrappedImage, &numreqs, reqs); + + // if we have a different number of aspwects, we can't apply this page table + if(numreqs != tables.size()) + { + rdcstr tablesStr, replayStr; + + for(size_t i = 0; i < tables.size(); i++) + tablesStr += ToStr((VkImageAspectFlagBits)tables[i].aspectMask) + ", "; + + if(tablesStr.size() > 2) + tablesStr.resize(tablesStr.size() - 2); + + for(uint32_t i = 0; i < numreqs; i++) + replayStr += ToStr((VkImageAspectFlagBits)reqs[i].formatProperties.aspectMask) + ", "; + + if(replayStr.size() > 2) + replayStr.resize(replayStr.size() - 2); + + RDCERR("Captured page table has %zu aspects (%s), but on replay we need %u aspects (%s)", + tables.size(), tablesStr.c_str(), numreqs, replayStr.c_str()); + invalid = true; + return; + } + + for(uint32_t a = 0; a < numreqs; a++) + { + // can't apply if the aspects mismatch + if(tables[a].aspectMask != reqs[a].formatProperties.aspectMask) + { + RDCERR("Captured page table aspect %u is %s, but on replay it is %s", a, + ToStr((VkImageAspectFlagBits)tables[a].aspectMask).c_str(), + ToStr((VkImageAspectFlagBits)reqs[a].formatProperties.aspectMask).c_str()); + invalid = true; + return; + } + + VkImageAspectFlags aspect = reqs[a].formatProperties.aspectMask; + const Sparse::PageTable &table = tables[a].table; + + // can't apply if page size changed + if(mrq.alignment != table.getPageByteSize()) + { + RDCERR("Captured page table for %s uses page size %llu, but on replay page size is %llu", + ToStr((VkImageAspectFlagBits)tables[a].aspectMask).c_str(), table.getPageByteSize(), + mrq.alignment); + invalid = true; + return; + } + + Sparse::Coord blockSize = table.getPageTexelSize(); + VkExtent3D gran = reqs[a].formatProperties.imageGranularity; + + // can't apply if the page texel dimension has changed + if(blockSize.x != RDCMAX(1U, gran.width) || blockSize.y != RDCMAX(1U, gran.height) || + blockSize.z != RDCMAX(1U, gran.depth)) + { + RDCERR("Captured page table for %s uses %ux%ux%u pages, but on replay pages are %ux%ux%u", + ToStr((VkImageAspectFlagBits)tables[a].aspectMask).c_str(), blockSize.x, blockSize.y, + blockSize.z, gran.width, gran.height, gran.depth); + invalid = true; + return; + } + + const Sparse::MipTail &mipTail = table.getMipTail(); + + // can't apply if the mip tail is differently shaped/sized + if(mipTail.byteOffset != reqs[a].imageMipTailOffset) + { + RDCERR("Captured mip tail for %s begins at offset %llu, on replay it begins at %llu", + ToStr((VkImageAspectFlagBits)tables[a].aspectMask).c_str(), mipTail.byteOffset, + reqs[a].imageMipTailOffset); + invalid = true; + return; + } + + if(mipTail.firstMip != RDCMIN(table.getMipCount(), reqs[a].imageMipTailFirstLod)) + { + RDCERR("Captured mip tail for %s begins at mip %u, on replay it begins at %u", + ToStr((VkImageAspectFlagBits)tables[a].aspectMask).c_str(), mipTail.firstMip, + reqs[a].imageMipTailFirstLod); + invalid = true; + return; + } + + if(reqs[a].formatProperties.flags & VK_SPARSE_IMAGE_FORMAT_SINGLE_MIPTAIL_BIT) + { + if(mipTail.totalPackedByteSize != reqs[a].imageMipTailSize) + { + RDCERR("Captured single mip tail for %s is %llu bytes, on replay it is %llu", + ToStr((VkImageAspectFlagBits)tables[a].aspectMask).c_str(), + mipTail.totalPackedByteSize, reqs[a].imageMipTailSize); + invalid = true; + return; + } + } + else + { + if(mipTail.totalPackedByteSize / table.getArraySize() != reqs[a].imageMipTailSize || + mipTail.byteStride != reqs[a].imageMipTailStride) + { + RDCERR( + "Captured mip tail per slice for %s is %llu bytes with stride %llu, " + "on replay it is %llu bytes with stride %llu", + ToStr((VkImageAspectFlagBits)tables[a].aspectMask).c_str(), + mipTail.totalPackedByteSize / table.getArraySize(), mipTail.byteStride, + reqs[a].imageMipTailSize, reqs[a].imageMipTailStride); + invalid = true; + return; + } + } + + { + VkSparseImageMemoryBind bind = {}; + bind.subresource.aspectMask = aspect; + if(aspect & VK_IMAGE_ASPECT_METADATA_BIT) + bind.flags = VK_SPARSE_MEMORY_BIND_METADATA_BIT; + + for(uint32_t slice = 0; slice < table.getArraySize(); slice++) + { + bind.subresource.arrayLayer = slice; + for(uint32_t mip = 0; mip < table.getMipCount(); mip++) + { + const uint32_t sub = table.calcSubresource(slice, mip); + + if(table.isSubresourceInMipTail(sub)) + continue; + + bind.subresource.mipLevel = mip; + + const Sparse::PageRangeMapping &mapping = table.getSubresource(sub); + + Sparse::Coord dim = table.calcSubresourcePageDim(sub); + + if(mapping.hasSingleMapping()) + { + // vulkan allows us to bind more than the subresource size as long as it's a case where + // we + // have less than a page used on the edges. + bind.offset = {}; + bind.extent.width = dim.x * blockSize.x; + bind.extent.height = dim.y * blockSize.y; + bind.extent.depth = dim.z * blockSize.z; + + bind.memory = Unwrap(vk->GetResourceManager()->GetLiveHandle( + mapping.singleMapping.memory)); + bind.memoryOffset = mapping.singleMapping.offset; + + imgBinds.push_back(bind); + } + else + { + uint32_t page = 0; + + // bind each block individually. Slow path :( + bind.extent = {blockSize.x, blockSize.y, blockSize.z}; + for(uint32_t z = 0; z < dim.z; z++) + { + bind.offset.z = z * blockSize.z; + for(uint32_t y = 0; y < dim.y; y++) + { + bind.offset.y = y * blockSize.y; + for(uint32_t x = 0; x < dim.x; x++) + { + bind.offset.x = x * blockSize.x; + + bind.memory = Unwrap(vk->GetResourceManager()->GetLiveHandle( + mapping.pages[page].memory)); + bind.memoryOffset = mapping.pages[page].offset; + + page++; + + imgBinds.push_back(bind); + } + } + } + } + } + } + } + + if(mipTail.totalPackedByteSize > 0) + { + VkSparseMemoryBind bind = {}; + + for(uint32_t slice = 0; slice < mipTail.mappings.size(); slice++) + { + const Sparse::PageRangeMapping &mapping = mipTail.mappings[slice]; + + // if all slices are in a combined mip tail, byteStride will be 0 and there will only be one + // mapping. Otherwise we look at each mapping individually and remap the resource offset as + // appropriate + + bind.resourceOffset = mipTail.byteOffset + mipTail.byteStride * slice; + + if(mapping.hasSingleMapping()) + { + bind.memory = Unwrap( + vk->GetResourceManager()->GetLiveHandle(mapping.singleMapping.memory)); + bind.memoryOffset = mapping.singleMapping.offset; + + // if stride is 0, we bind the whole mip tail at once. Otherwise only bind the section of + // memory backing it + if(mipTail.byteStride == 0) + bind.size = mipTail.totalPackedByteSize; + else + bind.size = mipTail.totalPackedByteSize / mipTail.mappings.size(); + + opaqueBinds.push_back(bind); + } + else + { + // bind a block at a time + bind.size = mrq.alignment; + + for(size_t i = 0; i < mapping.pages.size(); i++) + { + bind.memory = Unwrap( + vk->GetResourceManager()->GetLiveHandle(mapping.pages[i].memory)); + bind.memoryOffset = mapping.pages[i].offset; + + opaqueBinds.push_back(bind); + bind.resourceOffset += bind.size; + } + } + } + } + } + + imgOpaqueBind.image = unwrappedImage; + imgOpaqueBind.bindCount = (uint32_t)opaqueBinds.size(); + imgOpaqueBind.pBinds = opaqueBinds.data(); + + imgBind.image = unwrappedImage; + imgBind.bindCount = (uint32_t)imgBinds.size(); + imgBind.pBinds = imgBinds.data(); + + imageOpaqueBindCount = 1; + pImageOpaqueBinds = &imgOpaqueBind; + imageBindCount = 1; + pImageBinds = &imgBind; +} + template bool WrappedVulkan::Serialise_InitialState(SerialiserType &ser, ResourceId id, VkResourceRecord *, const VkInitialContents *initial) @@ -892,6 +1258,7 @@ bool WrappedVulkan::Serialise_InitialState(SerialiserType &ser, ResourceId id, V } else if(type == eResBuffer) { + // check for legacy captures if(ser.IsReading() && ser.VersionLess(0x13)) { RDCWARN( @@ -922,19 +1289,51 @@ bool WrappedVulkan::Serialise_InitialState(SerialiserType &ser, ResourceId id, V return true; } + + rdcarray sparseTables; + + // while writing, fetch sparse table from prepared initial contents + if(ser.IsWriting()) + { + sparseTables = *initial->sparseTables; + } + + SERIALISE_ELEMENT(sparseTables); + + SERIALISE_CHECK_READ_ERRORS(); + + // while reading, store the bindings in initial contents + if(IsReplayingAndReading()) + { + VkInitialContents initialContents(type, VkInitialContents::SparseTableOnly); + + WrappedVkRes *res = GetResourceManager()->GetLiveResource(id); + initialContents.sparseBind = + new SparseBinding(this, ToUnwrappedHandle(res), sparseTables); + + // if something went wrong the sparse binding information is not valid, have to abort + if(initialContents.sparseBind->invalid) + { + delete initialContents.sparseBind; + return false; + } + + GetResourceManager()->SetInitialContents(id, initialContents); + } } else if(type == eResDeviceMemory || type == eResImage) { VkDevice d = !IsStructuredExporting(m_State) ? GetDev() : VK_NULL_HANDLE; - // if we have a blob of data, this contains sparse mapping so re-direct to the sparse - // implementation of this function - SERIALISE_ELEMENT_LOCAL(IsSparse, false); + SERIALISE_ELEMENT_LOCAL(IsSparse, initial->sparseTables != NULL); + + rdcarray sparseTables; if(IsSparse) { if(type == eResImage) { + // check for legacy captures if(ser.IsReading() && ser.VersionLess(0x13)) { RDCWARN( @@ -981,6 +1380,12 @@ bool WrappedVulkan::Serialise_InitialState(SerialiserType &ser, ResourceId id, V return true; } + + // while writing, fetch sparse table from prepared initial contents + if(ser.IsWriting()) + sparseTables = *initial->sparseTables; + + SERIALISE_ELEMENT(sparseTables); } else { @@ -1097,6 +1502,21 @@ bool WrappedVulkan::Serialise_InitialState(SerialiserType &ser, ResourceId id, V { VkInitialContents initialContents(type, uploadMemory); + // if we have sparse page tables, store them here now + if(!sparseTables.empty()) + { + WrappedVkRes *res = GetResourceManager()->GetLiveResource(id); + initialContents.sparseBind = + new SparseBinding(this, ToUnwrappedHandle(res), sparseTables); + + // if something went wrong the sparse binding information is not valid, have to abort + if(initialContents.sparseBind->invalid) + { + delete initialContents.sparseBind; + return false; + } + } + VulkanCreationInfo::Image &c = m_CreationInfo.m_Image[liveid]; // for non-MSAA images, we're done - we'll do buffer-to-image copies with appropriate @@ -1335,10 +1755,10 @@ void WrappedVulkan::Create_InitialState(ResourceId id, WrappedVkRes *live, bool) } } -std::map > GetExtQBarriers( +std::map> GetExtQBarriers( const rdcarray &barriers) { - std::map > extQBarriers; + std::map> extQBarriers; for(auto barrierIt = barriers.begin(); barrierIt != barriers.end(); ++barrierIt) { @@ -1416,6 +1836,11 @@ void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, const VkInitialConten } else if(type == eResBuffer) { + // we should only get here if we have a sparse page table to apply + RDCASSERT(initial.tag == VkInitialContents::SparseTableOnly, (uint32_t)initial.tag); + + if(initial.sparseBind) + ObjDisp(m_Queue)->QueueBindSparse(Unwrap(m_Queue), 1, initial.sparseBind, VK_NULL_HANDLE); } else if(type == eResImage) { @@ -1441,7 +1866,13 @@ void WrappedVulkan::Apply_InitialState(WrappedVkRes *live, const VkInitialConten const ImageInfo &imageInfo = state->GetImageInfo(); initialized = IsActiveReplaying(m_State); - if(initialized && boundMemory != ResourceId()) + // apply sparse page table mappings and skip memory-bound optimisations + if(initial.sparseBind) + { + ObjDisp(m_Queue)->QueueBindSparse(Unwrap(m_Queue), 1, initial.sparseBind, VK_NULL_HANDLE); + initialized = false; + } + else if(initialized && boundMemory != ResourceId()) { ResourceId origMem = GetResourceManager()->GetOriginalID(boundMemory); if(origMem != ResourceId()) diff --git a/renderdoc/driver/vulkan/vk_manager.h b/renderdoc/driver/vulkan/vk_manager.h index 39aab0aac..0f85bf555 100644 --- a/renderdoc/driver/vulkan/vk_manager.h +++ b/renderdoc/driver/vulkan/vk_manager.h @@ -29,6 +29,23 @@ class WrappedVulkan; +struct SparseBinding : public VkBindSparseInfo +{ + SparseBinding(WrappedVulkan *vk, VkBuffer unwrappedBuffer, + const rdcarray &tables); + SparseBinding(WrappedVulkan *vk, VkImage unwrappedImage, const rdcarray &tables); + SparseBinding(const SparseBinding &) = delete; + + bool invalid = false; + + VkSparseBufferMemoryBindInfo bufBind; + VkSparseImageMemoryBindInfo imgBind; + VkSparseImageOpaqueMemoryBindInfo imgOpaqueBind; + + rdcarray opaqueBinds; + rdcarray imgBinds; +}; + // this struct is copied around and for that reason we explicitly keep it simple and POD. The // lifetime of the memory allocated is controlled by the resource manager - when preparing or // serialising, we explicitly set the initial contents, then when the whole system is done with them @@ -41,6 +58,7 @@ struct VkInitialContents ClearColorImage = 1, ClearDepthStencilImage, DescriptorSet, + SparseTableOnly, }; VkInitialContents() @@ -64,6 +82,20 @@ struct VkInitialContents mem = m; } + void SnapshotPageTable(const ResourceInfo &resInfo) + { + SAFE_DELETE(sparseTables); + + sparseTables = new rdcarray; + sparseTables->resize(resInfo.altSparseAspects.size() + 1); + + sparseTables->at(0).aspectMask = resInfo.sparseAspect; + sparseTables->at(0).table = resInfo.sparseTable; + + for(size_t a = 0; a < resInfo.altSparseAspects.size(); a++) + sparseTables->at(a + 1) = resInfo.altSparseAspects[a]; + } + template void Free(ResourceManager *rm) { @@ -77,7 +109,10 @@ struct VkInitialContents rm->ResourceTypeRelease(GetWrapped(buf)); rm->ResourceTypeRelease(GetWrapped(img)); - // memory is not free'd here + SAFE_DELETE(sparseTables); + SAFE_DELETE(sparseBind); + + // MemoryAllocation is not free'd here } // for descriptor heaps, when capturing we save the slots, when replaying we store direct writes @@ -95,6 +130,11 @@ struct VkInitialContents VkImage img; MemoryAllocation mem; Tag tag; + + // for sparse resources. The tables pointer is only valid on capture, it is converted to the queue + // sparse bind. Similar to the descriptors above + rdcarray *sparseTables; + SparseBinding *sparseBind; }; struct VulkanResourceManagerConfiguration @@ -165,10 +205,7 @@ public: ResourceId GetFirstIDForHandle(uint64_t handle); - // easy path for getting the unwrapped handle cast to the - // write type. Saves a lot of work casting to either WrappedVkNonDispRes - // or WrappedVkDispRes depending on the type, then ->real, then casting - // when this is all we want to do in most cases + // easy path for getting the wrapped handle cast to the correct type template realtype GetLiveHandle(ResourceId origid) {