From 4e37c99d53d7da8c3a669599d77e102cada05e25 Mon Sep 17 00:00:00 2001 From: Amit Prakash Date: Tue, 12 Jul 2022 22:01:05 -0400 Subject: [PATCH] Add RT buffer management and resource handling helpers * Add D3D12RaytracingResourceAndUtilHandler for managing DXR additional resources and workload * Add D3D12GpuBufferPool to manage GPU buffer resources --- renderdoc/driver/d3d12/d3d12_manager.cpp | 474 +++++++++++++++++++++++ renderdoc/driver/d3d12/d3d12_manager.h | 347 +++++++++++++++++ 2 files changed, 821 insertions(+) diff --git a/renderdoc/driver/d3d12/d3d12_manager.cpp b/renderdoc/driver/d3d12/d3d12_manager.cpp index 7a9277fe4..7cb09e7c5 100644 --- a/renderdoc/driver/d3d12/d3d12_manager.cpp +++ b/renderdoc/driver/d3d12/d3d12_manager.cpp @@ -682,6 +682,468 @@ D3D12Descriptor *DescriptorFromPortableHandle(D3D12ResourceManager *manager, Por #define BARRIER_ASSERT(...) #endif +D3D12RaytracingResourceAndUtilHandler::D3D12RaytracingResourceAndUtilHandler(WrappedID3D12Device *device) + : m_wrappedDevice(device), + m_cmdList(NULL), + m_cmdAlloc(NULL), + m_cmdQueue(NULL), + m_gpuFence(NULL), + m_gpuSyncHandle(NULL), + m_gpuSyncCounter(0u) +{ + if(m_wrappedDevice) + { + ID3D12Device *realDevice = m_wrappedDevice->GetReal(); + + if(realDevice) + { + HRESULT result = realDevice->CreateCommandAllocator( + D3D12_COMMAND_LIST_TYPE_DIRECT, __uuidof(ID3D12CommandAllocator), (void **)&m_cmdAlloc); + + if(!SUCCEEDED(result)) + RDCERR("D3D12 Command allocator creation failed with error %s", ToStr(result).c_str()); + + if(m_cmdAlloc != NULL) + { + ID3D12GraphicsCommandList *cmd = NULL; + result = realDevice->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, m_cmdAlloc, NULL, + __uuidof(ID3D12GraphicsCommandList), (void **)(&cmd)); + + if(!SUCCEEDED(result)) + RDCERR("D3D12 Command list creation failed with error %s", ToStr(result).c_str()); + + m_cmdList = (ID3D12GraphicsCommandListX *)cmd; + m_cmdList->Close(); + } + + D3D12_COMMAND_QUEUE_DESC cmdQueueDesc; + cmdQueueDesc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT; + cmdQueueDesc.NodeMask = 0; + cmdQueueDesc.Priority = D3D12_COMMAND_QUEUE_PRIORITY_NORMAL; + cmdQueueDesc.Flags = D3D12_COMMAND_QUEUE_FLAG_NONE; + result = realDevice->CreateCommandQueue(&cmdQueueDesc, __uuidof(ID3D12CommandQueue), + (void **)&m_cmdQueue); + + if(!SUCCEEDED(result)) + RDCERR("D3D12 Command queue creation failed with error %s", ToStr(result).c_str()); + + result = realDevice->CreateFence(0, D3D12_FENCE_FLAG_NONE, __uuidof(ID3D12Fence), + (void **)&m_gpuFence); + + if(!SUCCEEDED(result)) + RDCERR("D3D12 fence creation failed with error %s", ToStr(result).c_str()); + + m_gpuSyncHandle = ::CreateEvent(NULL, FALSE, FALSE, NULL); + } + } +} + +void D3D12RaytracingResourceAndUtilHandler::SyncGpuForRtWork() +{ + m_gpuSyncCounter++; + + HRESULT hr = m_cmdQueue->Signal(m_gpuFence, m_gpuSyncCounter); + if(!SUCCEEDED(hr)) + RDCERR("Command queue fence signaling failed with error %s ", ToStr(hr).c_str()); + + hr = m_gpuFence->SetEventOnCompletion(m_gpuSyncCounter, m_gpuSyncHandle); + if(!SUCCEEDED(hr)) + RDCERR("Fence completion event signaling failed with error %s ", ToStr(hr).c_str()); + + WaitForSingleObject(m_gpuSyncHandle, 10000); +} + +D3D12GpuBufferAllocator *D3D12GpuBufferAllocator::m_bufferAllocator = NULL; + +bool D3D12GpuBufferAllocator::CopyBufferRegion(WrappedID3D12GraphicsCommandList *wrappedCmd, + const D3D12GpuBuffer &destBuffer, + D3D12_GPU_VIRTUAL_ADDRESS srcAddress, + uint64_t dataSize) +{ + if(D3D12GpuBuffer() != destBuffer && dataSize > 0) + { + ResourceId srcResourceId; + D3D12BufferOffset srcResourceOffset; + + rdcarray resBarriers; + rdcarray finalBarriers; + + { + D3D12_RESOURCE_BARRIER resBarrier; + resBarrier.Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE; + resBarrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + resBarrier.Transition.StateBefore = D3D12_RESOURCE_STATE_COMMON; + resBarrier.Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_DEST; + resBarrier.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES; + resBarrier.Transition.pResource = destBuffer.Resource(); + resBarriers.push_back(resBarrier); + + resBarrier.Transition.StateBefore = D3D12_RESOURCE_STATE_COPY_DEST; + resBarrier.Transition.StateAfter = D3D12_RESOURCE_STATE_COMMON; + finalBarriers.push_back(resBarrier); + } + + WrappedID3D12Resource::GetResIDFromAddr(srcAddress, srcResourceId, srcResourceOffset); + + if(srcResourceId != ResourceId()) + { + D3D12_RESOURCE_STATES srResourceState = + wrappedCmd->GetWrappedDevice()->GetSubresourceStates(srcResourceId)[0].ToStates(); + + ID3D12Resource *srcResource = NULL; + srcResource = wrappedCmd->GetWrappedDevice() + ->GetResourceManager() + ->GetCurrentAs(srcResourceId) + ->GetReal(); + + if(!(srResourceState & D3D12_RESOURCE_STATE_COPY_SOURCE)) + { + D3D12_RESOURCE_BARRIER resBarrier; + resBarrier.Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE; + resBarrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + resBarrier.Transition.StateBefore = srResourceState; + resBarrier.Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_SOURCE; + resBarrier.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES; + resBarrier.Transition.pResource = srcResource; + resBarriers.push_back(resBarrier); + + resBarrier.Transition.StateBefore = D3D12_RESOURCE_STATE_COPY_SOURCE; + resBarrier.Transition.StateAfter = srResourceState; + finalBarriers.push_back(resBarrier); + } + + wrappedCmd->GetReal()->ResourceBarrier((UINT)resBarriers.size(), resBarriers.data()); + wrappedCmd->GetReal()->CopyBufferRegion(destBuffer.Resource(), destBuffer.Offset(), + srcResource, srcResourceOffset, dataSize); + wrappedCmd->GetReal()->ResourceBarrier((UINT)finalBarriers.size(), finalBarriers.data()); + + return true; + } + } + + return false; +} + +bool D3D12GpuBufferAllocator::CopyBufferRegion(WrappedID3D12GraphicsCommandList *wrappedCmd, + const D3D12GpuBuffer &destBuffer, + const D3D12GpuBuffer &sourceBuffer, uint64_t dataSize) +{ + // This will only handle if both are on default heap + if(destBuffer.GetD3D12HeapType() != D3D12_HEAP_TYPE_DEFAULT || + sourceBuffer.GetD3D12HeapType() != D3D12_HEAP_TYPE_DEFAULT) + { + return false; + } + + rdcarray initBarriers; + rdcarray finalBarriers; + + { + D3D12_RESOURCE_BARRIER resBarrier; + resBarrier.Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE; + resBarrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + resBarrier.Transition.StateBefore = D3D12_RESOURCE_STATE_COMMON; + resBarrier.Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_SOURCE; + resBarrier.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES; + resBarrier.Transition.pResource = sourceBuffer.Resource(); + initBarriers.push_back(resBarrier); + + resBarrier.Transition.StateBefore = D3D12_RESOURCE_STATE_COPY_SOURCE; + resBarrier.Transition.StateAfter = D3D12_RESOURCE_STATE_COMMON; + finalBarriers.push_back(resBarrier); + } + + { + D3D12_RESOURCE_BARRIER resBarrier; + resBarrier.Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE; + resBarrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + resBarrier.Transition.StateBefore = D3D12_RESOURCE_STATE_COMMON; + resBarrier.Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_DEST; + resBarrier.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES; + resBarrier.Transition.pResource = destBuffer.Resource(); + initBarriers.push_back(resBarrier); + + resBarrier.Transition.StateBefore = D3D12_RESOURCE_STATE_COPY_DEST; + resBarrier.Transition.StateAfter = D3D12_RESOURCE_STATE_COMMON; + finalBarriers.push_back(resBarrier); + } + + wrappedCmd->GetReal()->ResourceBarrier((UINT)initBarriers.size(), initBarriers.data()); + wrappedCmd->GetReal()->CopyBufferRegion(destBuffer.Resource(), destBuffer.Offset(), + sourceBuffer.Resource(), sourceBuffer.Offset(), dataSize); + wrappedCmd->GetReal()->ResourceBarrier((UINT)finalBarriers.size(), finalBarriers.data()); + return true; +} + +bool D3D12GpuBufferAllocator::D3D12GpuBufferResource::CreateCommittedResourceBuffer( + ID3D12Device *device, const D3D12_HEAP_PROPERTIES &heapProperty, D3D12_RESOURCE_STATES initState, + uint64_t size, bool allowUav, D3D12GpuBufferResource **bufferResource) +{ + if(device && bufferResource) + { + ID3D12Resource *newBufferResource = NULL; + + D3D12_RESOURCE_DESC bufferResDesc; + bufferResDesc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; + bufferResDesc.Alignment = D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT; + bufferResDesc.DepthOrArraySize = 1u; + bufferResDesc.MipLevels = 1u; + bufferResDesc.Height = 1u; + bufferResDesc.Flags = + allowUav ? D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS : D3D12_RESOURCE_FLAG_NONE; + bufferResDesc.Format = DXGI_FORMAT_UNKNOWN; + bufferResDesc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; + bufferResDesc.SampleDesc = {1, 0}; + bufferResDesc.Width = size; + + D3D12GpuBufferResource *retBufferRes = NULL; + + // Create committed resource + HRESULT opResult = device->CreateCommittedResource( + &heapProperty, D3D12_HEAP_FLAG_NONE, &bufferResDesc, initState, NULL, + __uuidof(ID3D12Resource), (void **)&newBufferResource); + + if(SUCCEEDED(opResult) && newBufferResource != NULL) + { + retBufferRes = new D3D12GpuBufferResource(newBufferResource, heapProperty.Type); + } + else + { + RDCERR("Allocation failed with result code %s", ToStr(opResult).c_str()); + } + + if(retBufferRes) + { + *bufferResource = retBufferRes; + return true; + } + } + + return false; +} + +bool D3D12GpuBufferAllocator::D3D12GpuBufferResource::ReleaseGpuBufferResource( + D3D12GpuBufferResource *bufferResource) +{ + delete bufferResource; + bufferResource = NULL; + return true; +} + +D3D12GpuBufferAllocator::D3D12GpuBufferResource::D3D12GpuBufferResource(ID3D12Resource *resource, + D3D12_HEAP_TYPE heapType) + : m_resource(resource), m_heapType(heapType) + +{ + if(m_resource) + { + m_resDesc = m_resource->GetDesc(); + m_resourceGpuAddressRange.start = resource->GetGPUVirtualAddress(); + m_resourceGpuAddressRange.realEnd = m_resourceGpuAddressRange.start + m_resDesc.Width; + } +} + +bool D3D12GpuBufferAllocator::D3D12GpuBufferPool::Alloc(WrappedID3D12Device *wrappedDevice, + D3D12GpuBufferHeapMemoryFlag heapMem, + uint64_t size, uint64_t alignment, + D3D12GpuBuffer &gpuBuffer) +{ + if(heapMem == D3D12GpuBufferHeapMemoryFlag::Default) + { + if(size > m_bufferInitSize) + { + m_bufferInitSize = size; + } + + D3D12_GPU_VIRTUAL_ADDRESS gpuAddress = 0; + for(D3D12GpuBufferResource *bufferRes : m_bufferResourceList) + { + if(bufferRes->SubAlloc(size, alignment, gpuAddress)) + { + gpuBuffer = D3D12GpuBuffer(m_bufferPoolHeapType, D3D12GpuBufferHeapMemoryFlag::Default, + size, alignment, gpuAddress, bufferRes->Resource()); + return true; + } + } + + D3D12GpuBufferResource *newBufferResource = NULL; + if(D3D12GpuBufferAllocator::CreateBufferResource(wrappedDevice, m_bufferPoolHeapType, + m_bufferInitSize, &newBufferResource)) + { + m_bufferResourceList.push_back(newBufferResource); + if(newBufferResource->SubAlloc(size, alignment, gpuAddress)) + { + gpuBuffer = D3D12GpuBuffer(m_bufferPoolHeapType, D3D12GpuBufferHeapMemoryFlag::Default, + size, alignment, gpuAddress, newBufferResource->Resource()); + return true; + } + } + } + else + { + D3D12GpuBufferResource *newBufferResource = NULL; + if(CreateBufferResource(m_bufferAllocator->m_wrappedDevice, m_bufferPoolHeapType, size, + &newBufferResource)) + { + m_bufferResourceList.push_back(newBufferResource); + gpuBuffer = D3D12GpuBuffer(m_bufferPoolHeapType, D3D12GpuBufferHeapMemoryFlag::Dedicated, + size, D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT, + newBufferResource->Resource()->GetGPUVirtualAddress(), + newBufferResource->Resource()); + return true; + } + } + + RDCERR("Unable to allocate GPU memory"); + return false; +} + +bool D3D12GpuBufferAllocator::D3D12GpuBufferPool::Free(const D3D12GpuBuffer &gpuBuffer) +{ + if(gpuBuffer != D3D12GpuBuffer()) + { + for(D3D12GpuBufferResource *bufferRes : m_bufferResourceList) + { + if(bufferRes->Resource() == gpuBuffer.Resource()) + { + D3D12GpuBufferHeapMemoryFlag heapMem = gpuBuffer.HeapMemory(); + if(heapMem == D3D12GpuBufferHeapMemoryFlag::Default) + { + if(bufferRes->SubAllocationInRange(gpuBuffer.Address())) + { + return bufferRes->Free(gpuBuffer.Address()); + } + } + else if(heapMem == D3D12GpuBufferHeapMemoryFlag::Dedicated) + { + if(D3D12GpuBufferResource::ReleaseGpuBufferResource(bufferRes)) + { + m_bufferResourceList.removeOne(bufferRes); + return true; + } + } + } + } + } + + return false; +} + +bool D3D12GpuBufferAllocator::Alloc(D3D12GpuBufferHeapType heapType, + D3D12GpuBufferHeapMemoryFlag heapMem, uint64_t size, + uint64_t alignment, D3D12GpuBuffer &gpuBuffer) +{ + SCOPED_LOCK(m_bufferAllocLock); + bool success = false; + if(heapType < D3D12GpuBufferHeapType::Count && heapType != D3D12GpuBufferHeapType::UnInitialized) + { + size_t heap = (size_t)heapType; + if(m_bufferPoolList[heap] == NULL) + { + uint64_t bufferPoolInitSize = D3D12GpuBufferPool::kDefaultWithUavSizeBufferInitSize; + if(heapType == D3D12GpuBufferHeapType::AccStructDefaultHeap) + { + bufferPoolInitSize = D3D12GpuBufferPool::kAccStructBufferPoolInitSize; + } + + m_bufferPoolList[heap] = new D3D12GpuBufferPool(heapType, bufferPoolInitSize); + } + + if(m_bufferPoolList[heap] != NULL) + { + success = m_bufferPoolList[heap]->Alloc(m_wrappedDevice, heapMem, size, alignment, gpuBuffer); + } + } + + if(success) + { + m_totalAllocatedMemoryInUse += size; + } + + return success; +} + +bool D3D12GpuBufferAllocator::Release(const D3D12GpuBuffer &gpuBuffer) +{ + SCOPED_LOCK(m_bufferAllocLock); + size_t heap = (size_t)gpuBuffer.HeapType(); + if(gpuBuffer.HeapType() < D3D12GpuBufferHeapType::Count && m_bufferPoolList[heap] != NULL) + { + return m_bufferPoolList[heap]->Free(gpuBuffer); + } + + return false; +} + +bool D3D12GpuBufferAllocator::CreateBufferResource(WrappedID3D12Device *wrappedDevice, + D3D12GpuBufferHeapType heapType, uint64_t size, + D3D12GpuBufferResource **bufferResource) +{ + D3D12_HEAP_PROPERTIES heapProperty; + heapProperty.CreationNodeMask = 0; + heapProperty.VisibleNodeMask = 0; + heapProperty.CPUPageProperty = D3D12_CPU_PAGE_PROPERTY_UNKNOWN; + heapProperty.MemoryPoolPreference = D3D12_MEMORY_POOL_UNKNOWN; + + D3D12_RESOURCE_STATES initState = D3D12_RESOURCE_STATE_COMMON; + bool allowUav = false; + switch(heapType) + { + case D3D12GpuBufferHeapType::AccStructDefaultHeap: + case D3D12GpuBufferHeapType::DefaultHeap: + case D3D12GpuBufferHeapType::DefaultHeapWithUav: + { + heapProperty.Type = D3D12_HEAP_TYPE_DEFAULT; + if(heapType == D3D12GpuBufferHeapType::AccStructDefaultHeap) + { + initState = D3D12_RESOURCE_STATE_RAYTRACING_ACCELERATION_STRUCTURE; + allowUav = true; + } + + if(heapType == D3D12GpuBufferHeapType::DefaultHeapWithUav) + { + allowUav = true; + } + + break; + } + + case D3D12GpuBufferHeapType::ReadBackHeap: + { + heapProperty.Type = D3D12_HEAP_TYPE_READBACK; + initState = D3D12_RESOURCE_STATE_COPY_DEST; + break; + } + + case D3D12GpuBufferHeapType::UploadHeap: + { + heapProperty.Type = D3D12_HEAP_TYPE_UPLOAD; + initState = D3D12_RESOURCE_STATE_GENERIC_READ; + break; + } + + case D3D12GpuBufferHeapType::CustomHeapWithUavCpuAccess: + { + heapProperty.Type = D3D12_HEAP_TYPE_CUSTOM; + heapProperty.CPUPageProperty = D3D12_CPU_PAGE_PROPERTY_WRITE_BACK; + heapProperty.MemoryPoolPreference = D3D12_MEMORY_POOL_L0; + allowUav = true; + break; + } + default: RDCLOG("Unhandled buffer pool"); + } + + D3D12GpuBufferResource *newBufferResource = NULL; + if(D3D12GpuBufferResource::CreateCommittedResourceBuffer( + wrappedDevice->GetReal(), heapProperty, initState, size, allowUav, &newBufferResource)) + { + *bufferResource = newBufferResource; + return true; + } + + return false; +} + void D3D12ResourceManager::ApplyBarriers(BarrierSet &barriers, std::map &states) { @@ -1081,3 +1543,15 @@ void GPUAddressRangeTracker::GetResIDFromAddrAllowOutOfBounds(D3D12_GPU_VIRTUAL_ id = range.id; offs = addr - range.start; } + +bool D3D12GpuBuffer::Release() +{ + bool success = D3D12GpuBufferAllocator::Inst()->Release(*this); + + if(success) + { + *this = {}; + } + + return success; +} diff --git a/renderdoc/driver/d3d12/d3d12_manager.h b/renderdoc/driver/d3d12/d3d12_manager.h index c330d255d..5bd16c824 100644 --- a/renderdoc/driver/d3d12/d3d12_manager.h +++ b/renderdoc/driver/d3d12/d3d12_manager.h @@ -26,6 +26,7 @@ #include "common/wrapped_pool.h" #include "core/core.h" +#include "core/intervals.h" #include "core/resource_manager.h" #include "core/sparse_page_table.h" #include "driver/d3d12/d3d12_common.h" @@ -502,6 +503,7 @@ struct CmdListRecordingInfo }; class WrappedID3D12Resource; +using D3D12BufferOffset = UINT64; struct GPUAddressRange { @@ -542,6 +544,110 @@ struct MapState bool operator==(const MapState &o) { return res == o.res && subres == o.subres; } }; +// Enum for the supported heap type for D3D12GpuBuffer allocation +enum class D3D12GpuBufferHeapType +{ + UnInitialized = 0, // Not initialized + AccStructDefaultHeap, // Buffer pool of resource with + // D3D12_RESOURCE_STATE_RAYTRACING_ACCELERATION_STRUCTURE + // init state + ReadBackHeap, // Buffer pool with resource on read back heap + UploadHeap, // Buffer Pool with resource on upload heap + DefaultHeap, // Buffer Pool with resource on default heap + DefaultHeapWithUav, // Buffer with resource on default heap with UAV enabled + CustomHeapWithUavCpuAccess, // Buffer Pool with resource on Custom heap with UAV and CPU access + Count +}; + +// Flag for the heap allocation to decide whether to sub-alloc or alloc a dedicated +// heap (currently only implicit heap from CommittedResource is supported) +enum class D3D12GpuBufferHeapMemoryFlag +{ + UnInitialized = 0, + Default, // Buffer will be sub-allocated, and heap will be shared with other + Dedicated, // Buffer will have a dedicated heap +}; + +class D3D12GpuBufferAllocator; + +struct D3D12GpuBuffer +{ + D3D12GpuBuffer() + : m_alignedAddress(0), + m_offset(0), + m_alignment(0), + m_addressContentSize(0), + m_heapType(D3D12GpuBufferHeapType::UnInitialized), + m_heapMemory(D3D12GpuBufferHeapMemoryFlag::UnInitialized), + m_resource(NULL) + { + } + + D3D12GpuBuffer(D3D12GpuBufferHeapType heapType, D3D12GpuBufferHeapMemoryFlag heapMemory, + uint64_t size, uint64_t alignment, D3D12_GPU_VIRTUAL_ADDRESS alignedAddress, + ID3D12Resource *resource) + : m_alignedAddress(alignedAddress), + m_offset(0), + m_alignment(alignment), + m_addressContentSize(size), + m_heapType(heapType), + m_heapMemory(heapMemory), + m_resource(resource) + { + if(m_resource) + { + m_offset = alignedAddress - m_resource->GetGPUVirtualAddress(); + } + } + + D3D12GpuBufferHeapType HeapType() const { return m_heapType; } + D3D12_HEAP_TYPE GetD3D12HeapType() const + { + switch(m_heapType) + { + case D3D12GpuBufferHeapType::AccStructDefaultHeap: + case D3D12GpuBufferHeapType::DefaultHeap: + case D3D12GpuBufferHeapType::DefaultHeapWithUav: return D3D12_HEAP_TYPE_DEFAULT; + case D3D12GpuBufferHeapType::ReadBackHeap: return D3D12_HEAP_TYPE_READBACK; + case D3D12GpuBufferHeapType::UploadHeap: return D3D12_HEAP_TYPE_UPLOAD; + case D3D12GpuBufferHeapType::CustomHeapWithUavCpuAccess: return D3D12_HEAP_TYPE_CUSTOM; + default: RDCERR("Unhandled/Invalid type"); + } + + return D3D12_HEAP_TYPE_DEFAULT; + } + bool operator!=(const D3D12GpuBuffer &other) const { return !(*this == other); } + bool operator==(const D3D12GpuBuffer &other) const + { + bool equal = true; + equal &= m_alignedAddress == other.m_alignedAddress; + equal &= m_alignment == other.m_alignment; + equal &= m_addressContentSize == other.m_addressContentSize; + equal &= m_heapType == other.m_heapType; + equal &= m_heapMemory == other.m_heapMemory; + equal &= m_resource == other.m_resource; + equal &= m_offset == other.m_offset; + + return equal; + } + + ID3D12Resource *Resource() const { return m_resource; }; + uint64_t Offset() const { return m_offset; } + uint64_t Size() const { return m_addressContentSize; } + D3D12_GPU_VIRTUAL_ADDRESS Address() const { return m_alignedAddress; } + uint64_t Alignment() const { return m_alignment; } + bool Release(); + D3D12GpuBufferHeapMemoryFlag HeapMemory() const { return m_heapMemory; } +private: + D3D12_GPU_VIRTUAL_ADDRESS m_alignedAddress; + uint64_t m_offset; + uint64_t m_alignment; + uint64_t m_addressContentSize; + D3D12GpuBufferHeapType m_heapType; + D3D12GpuBufferHeapMemoryFlag m_heapMemory; + ID3D12Resource *m_resource; +}; + struct D3D12ResourceRecord : public ResourceRecord { enum @@ -733,6 +839,232 @@ struct D3D12InitialContents SparseBinds *sparseBinds; }; +class WrappedID3D12GraphicsCommandList; + +// class for allocating GPU Buffer +class D3D12GpuBufferAllocator +{ +public: + static bool Initialize(WrappedID3D12Device *wrappedDevice) + { + if(m_bufferAllocator == NULL && wrappedDevice) + { + m_bufferAllocator = new D3D12GpuBufferAllocator(wrappedDevice); + } + + return m_bufferAllocator != NULL; + } + + static D3D12GpuBufferAllocator *Inst() { return m_bufferAllocator; } + static bool Destroy() + { + SAFE_DELETE(m_bufferAllocator); + return true; + } + + static bool CopyBufferRegion(WrappedID3D12GraphicsCommandList *wrappedCmd, + const D3D12GpuBuffer &destBuffer, + D3D12_GPU_VIRTUAL_ADDRESS srcAddress, uint64_t dataSize); + + static bool CopyBufferRegion(WrappedID3D12GraphicsCommandList *wrappedCmd, + const D3D12GpuBuffer &destBuffer, const D3D12GpuBuffer &sourceBuffer, + uint64_t dataSize); + + bool Alloc(D3D12GpuBufferHeapType heapType, D3D12GpuBufferHeapMemoryFlag heapMem, uint64_t size, + D3D12GpuBuffer &gpuBuffer) + { + return Alloc(heapType, heapMem, size, 0, gpuBuffer); + } + + bool Alloc(D3D12GpuBufferHeapType heapType, D3D12GpuBufferHeapMemoryFlag heapMem, uint64_t size, + uint64_t alignment, D3D12GpuBuffer &gpuBuffer); + + bool Release(const D3D12GpuBuffer &gpuBuffer); + + uint64_t GetAllocatedMemorySize() const { return m_totalAllocatedMemoryInUse; } + ~D3D12GpuBufferAllocator() + { + for(D3D12GpuBufferPool *bufferPool : m_bufferPoolList) + { + SAFE_DELETE(bufferPool); + } + } + +private: + D3D12GpuBufferAllocator(WrappedID3D12Device *wrappedDevice) : m_wrappedDevice(wrappedDevice) + { + m_totalAllocatedMemoryInUse = 0; + } + + // Class for handling buffer resources + class D3D12GpuBufferResource + { + public: + static bool CreateCommittedResourceBuffer(ID3D12Device *device, + const D3D12_HEAP_PROPERTIES &heapProperty, + D3D12_RESOURCE_STATES initState, uint64_t size, + bool allowUav, D3D12GpuBufferResource **bufferResource); + static bool ReleaseGpuBufferResource(D3D12GpuBufferResource *bufferResource); + + D3D12GpuBufferResource() = delete; + + ID3D12Resource *Resource() const { return m_resource; } + ~D3D12GpuBufferResource() { SAFE_RELEASE(m_resource); } + D3D12GpuBufferResource(ID3D12Resource *resource, D3D12_HEAP_TYPE heapType); + + bool SubAllocationInRange(D3D12_GPU_VIRTUAL_ADDRESS gpuAddress) const + { + if(m_resourceGpuAddressRange.start <= gpuAddress && + gpuAddress < m_resourceGpuAddressRange.realEnd) + + { + return true; + } + + return false; + } + + bool Free(D3D12_GPU_VIRTUAL_ADDRESS gpuAddress) + { + uint64_t offset = gpuAddress - m_resourceGpuAddressRange.start; + auto iter = m_subRanges.find(offset); + if(iter != m_subRanges.end() && iter->value() == D3D12SubRangeFlag::Used) + { + iter->setValue(D3D12SubRangeFlag::Free); + // Merging will only occur if the adjacent sub-ranges are also free + iter->mergeLeft(); + ++iter; + if(iter != m_subRanges.end()) + iter->mergeLeft(); + return true; + } + return false; + } + + bool SubAlloc(uint64_t size, uint64_t alignment, D3D12_GPU_VIRTUAL_ADDRESS &address) + { + uint64_t resourceWidth = m_resourceGpuAddressRange.realEnd - m_resourceGpuAddressRange.start; + + for(auto iter = m_subRanges.begin(); iter != m_subRanges.end(); ++iter) + { + if(iter->value() == D3D12SubRangeFlag::Free) + { + uint64_t addr = iter->start() + m_resourceGpuAddressRange.start; + uint64_t end = RDCMIN(iter->finish(), resourceWidth) + m_resourceGpuAddressRange.start; + uint64_t alignedAddr = alignment != 0 ? AlignUp(addr, alignment) : addr; + + if(alignedAddr < end && size <= (end - alignedAddr)) + { + uint64_t offset = alignedAddr - m_resourceGpuAddressRange.start; + // Free the extra space from aligning + if(alignedAddr > addr) + { + iter->split(offset); + } + + iter->setValue(D3D12SubRangeFlag::Used); + address = alignedAddr; + // Split the sub-range if there's extra space beyond this allocation + if(size < (end - alignedAddr)) + { + iter->split(offset + size); + iter->setValue(D3D12SubRangeFlag::Free); + } + return true; + } + } + } + return false; + } + + enum class D3D12SubRangeFlag + { + Free = 0, + Used + }; + + Intervals m_subRanges; + GPUAddressRange m_resourceGpuAddressRange; + ID3D12Resource *m_resource; + D3D12_RESOURCE_DESC m_resDesc; + D3D12_HEAP_TYPE m_heapType; + }; + + class D3D12GpuBufferPool + { + public: + D3D12GpuBufferPool(D3D12GpuBufferHeapType bufferPoolType, uint64_t bufferInitialSize) + : m_bufferPoolHeapType(bufferPoolType), m_bufferInitSize(bufferInitialSize) + { + } + + ~D3D12GpuBufferPool() + { + for(D3D12GpuBufferResource *bufferRes : m_bufferResourceList) + { + SAFE_DELETE(bufferRes); + } + } + + bool Alloc(WrappedID3D12Device *wrappedDevice, D3D12GpuBufferHeapMemoryFlag heapMem, + uint64_t size, uint64_t alignment, D3D12GpuBuffer &gpuBuffer); + + bool Free(const D3D12GpuBuffer &gpuBuffer); + + static constexpr uint64_t kDefaultWithUavSizeBufferInitSize = 1000ull * 8u; + static constexpr uint64_t kAccStructBufferPoolInitSize = 1000ull * 256u; + + private: + rdcarray m_bufferResourceList; + D3D12GpuBufferHeapType m_bufferPoolHeapType; + uint64_t m_bufferInitSize; + }; + + static D3D12GpuBufferAllocator *m_bufferAllocator; + + static bool CreateBufferResource(WrappedID3D12Device *wrappedDevice, + D3D12GpuBufferHeapType heapType, uint64_t size, + D3D12GpuBufferResource **bufferResource); + + Threading::CriticalSection m_bufferAllocLock; + D3D12GpuBufferPool *m_bufferPoolList[(size_t)D3D12GpuBufferHeapType::Count] = {}; + + WrappedID3D12Device *m_wrappedDevice; + // keeps track of the allocated memory in use, + // and not the actual amount of memory allocated + uint64_t m_totalAllocatedMemoryInUse; +}; + +class D3D12RaytracingResourceAndUtilHandler +{ +public: + D3D12RaytracingResourceAndUtilHandler(WrappedID3D12Device *device); + + ID3D12GraphicsCommandListX *GetCmd() const { return m_cmdList; } + ID3D12CommandAllocator *GetCmdAlloc() const { return m_cmdAlloc; } + ID3D12CommandQueue *GetCmdQueue() const { return m_cmdQueue; } + ID3D12Fence *GetFence() const { return m_gpuFence; } + void SyncGpuForRtWork(); + + ~D3D12RaytracingResourceAndUtilHandler() + { + SAFE_RELEASE(m_cmdList); + SAFE_RELEASE(m_cmdAlloc); + SAFE_RELEASE(m_cmdQueue); + SAFE_RELEASE(m_gpuFence); + } + +private: + WrappedID3D12Device *m_wrappedDevice; + + ID3D12GraphicsCommandListX *m_cmdList; + ID3D12CommandAllocator *m_cmdAlloc; + ID3D12CommandQueue *m_cmdQueue; + ID3D12Fence *m_gpuFence; + HANDLE m_gpuSyncHandle; + UINT64 m_gpuSyncCounter; +}; + struct D3D12ResourceManagerConfiguration { typedef ID3D12DeviceChild *WrappedResourceType; @@ -747,7 +1079,16 @@ public: D3D12ResourceManager(CaptureState &state, WrappedID3D12Device *dev) : ResourceManager(state), m_Device(dev) { + m_raytracingResourceManager = new D3D12RaytracingResourceAndUtilHandler(m_Device); + D3D12GpuBufferAllocator::Initialize(dev); } + + ~D3D12ResourceManager() + { + D3D12GpuBufferAllocator::Destroy(); + SAFE_DELETE(m_raytracingResourceManager); + } + template T *GetLiveAs(ResourceId id, bool optional = false) { @@ -762,6 +1103,11 @@ public: void ApplyBarriers(BarrierSet &barriers, std::map &states); + D3D12RaytracingResourceAndUtilHandler *GetRaytracingResourceAndUtilHandler() const + { + return m_raytracingResourceManager; + } + template void SerialiseResourceStates(SerialiserType &ser, BarrierSet &barriers, std::map &states, @@ -789,4 +1135,5 @@ private: void Apply_InitialState(ID3D12DeviceChild *live, const D3D12InitialContents &data); WrappedID3D12Device *m_Device; + D3D12RaytracingResourceAndUtilHandler *m_raytracingResourceManager; };