From f20c596a695869f7a2d0d37f2de1e52fe8e7f3b7 Mon Sep 17 00:00:00 2001 From: Steve Karolewics Date: Sat, 16 Dec 2023 14:17:55 -0800 Subject: [PATCH] Fix D3D12 pixel history for events that do not have a pixel shader --- renderdoc/driver/d3d12/d3d12_pixelhistory.cpp | 20 +++++++++++------- .../driver/shaders/dxbc/dxbc_container.cpp | 21 ++++++++++++++++--- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/renderdoc/driver/d3d12/d3d12_pixelhistory.cpp b/renderdoc/driver/d3d12/d3d12_pixelhistory.cpp index 7c3988119..4880ae197 100644 --- a/renderdoc/driver/d3d12/d3d12_pixelhistory.cpp +++ b/renderdoc/driver/d3d12/d3d12_pixelhistory.cpp @@ -511,6 +511,15 @@ protected: } } + bool IsPSOUsingDXIL(D3D12_EXPANDED_PIPELINE_STATE_STREAM_DESC &pipeDesc) + { + // Check early stages first, for both VS and MS, but use PS as a fallback. If any shader stage + // uses DXIL, they all need to use DXIL. + return DXBC::DXBCContainer::CheckForDXIL(pipeDesc.VS.pShaderBytecode, pipeDesc.VS.BytecodeLength) || + DXBC::DXBCContainer::CheckForDXIL(pipeDesc.MS.pShaderBytecode, pipeDesc.MS.BytecodeLength) || + DXBC::DXBCContainer::CheckForDXIL(pipeDesc.PS.pShaderBytecode, pipeDesc.PS.BytecodeLength); + } + // ModifyPSOForStencilIncrement modifies the provided pipeDesc, by disabling depth test // and write, stencil is set to always pass and increment, scissor is set to scissor around // the target pixel, and all color modifications are disabled. @@ -830,8 +839,7 @@ private: ModifyPSOForStencilIncrement(eid, pipeDesc, true); - bool dxil = - DXBC::DXBCContainer::CheckForDXIL(pipeDesc.PS.pShaderBytecode, pipeDesc.PS.BytecodeLength); + bool dxil = IsPSOUsingDXIL(pipeDesc); ID3DBlob *psBlob = m_pDevice->GetShaderCache()->MakeFixedColShader(D3D12ShaderCache::HIGHLIGHT, dxil); @@ -1146,7 +1154,7 @@ private: D3D12_EXPANDED_PIPELINE_STATE_STREAM_DESC desc; origPSO->Fill(desc); - bool dxil = DXBC::DXBCContainer::CheckForDXIL(desc.PS.pShaderBytecode, desc.PS.BytecodeLength); + bool dxil = IsPSOUsingDXIL(desc); ID3DBlob *psBlob = m_pDevice->GetShaderCache()->MakeFixedColShader(D3D12ShaderCache::HIGHLIGHT, dxil); if(psBlob == NULL) @@ -1683,8 +1691,7 @@ private: if(pipeCreateFlags & PipelineCreationFlags_FixedColorShader) { - bool dxil = - DXBC::DXBCContainer::CheckForDXIL(pipeDesc.PS.pShaderBytecode, pipeDesc.PS.BytecodeLength); + bool dxil = IsPSOUsingDXIL(pipeDesc); ID3DBlob *FixedColorPS = m_ShaderCache->GetFixedColorShader(dxil); pipeDesc.PS.pShaderBytecode = FixedColorPS->GetBufferPointer(); @@ -2131,8 +2138,7 @@ struct D3D12PixelHistoryPerFragmentCallback : D3D12PixelHistoryCallback } else { - bool dxil = - DXBC::DXBCContainer::CheckForDXIL(pipeDesc.PS.pShaderBytecode, pipeDesc.PS.BytecodeLength); + bool dxil = IsPSOUsingDXIL(pipeDesc); // TODO: This shader outputs to SV_Target0, will we need to modify the // shader (or patch it) if the target image isn't RT0? diff --git a/renderdoc/driver/shaders/dxbc/dxbc_container.cpp b/renderdoc/driver/shaders/dxbc/dxbc_container.cpp index 3242672fc..1ad3fe8b2 100644 --- a/renderdoc/driver/shaders/dxbc/dxbc_container.cpp +++ b/renderdoc/driver/shaders/dxbc/dxbc_container.cpp @@ -470,6 +470,9 @@ D3D_PRIMITIVE_TOPOLOGY DXBCContainer::GetOutputTopology(const void *ByteCode, si const byte *data = (const byte *)ByteCode; // just for convenience + if(ByteCode == NULL || ByteCodeLength == 0) + return D3D_PRIMITIVE_TOPOLOGY_UNDEFINED; + if(header->fourcc != FOURCC_DXBC) return D3D_PRIMITIVE_TOPOLOGY_UNDEFINED; @@ -776,7 +779,7 @@ const byte *DXBCContainer::FindChunk(const bytebuf &ByteCode, uint32_t fourcc, s void DXBCContainer::GetHash(uint32_t hash[4], const void *ByteCode, size_t BytecodeLength) { - if(BytecodeLength < sizeof(FileHeader)) + if(BytecodeLength < sizeof(FileHeader) || ByteCode == NULL) { memset(hash, 0, sizeof(uint32_t) * 4); return; @@ -816,7 +819,7 @@ void DXBCContainer::GetHash(uint32_t hash[4], const void *ByteCode, size_t Bytec bool DXBCContainer::IsHashedContainer(const void *ByteCode, size_t BytecodeLength) { - if(BytecodeLength < sizeof(FileHeader)) + if(BytecodeLength < sizeof(FileHeader) || ByteCode == NULL) return false; const FileHeader *header = (const FileHeader *)ByteCode; @@ -836,7 +839,7 @@ bool DXBCContainer::IsHashedContainer(const void *ByteCode, size_t BytecodeLengt bool DXBCContainer::HashContainer(void *ByteCode, size_t BytecodeLength) { - if(BytecodeLength < sizeof(FileHeader)) + if(BytecodeLength < sizeof(FileHeader) || ByteCode == NULL) return false; FileHeader *header = (FileHeader *)ByteCode; @@ -943,6 +946,9 @@ bool DXBCContainer::UsesExtensionUAV(uint32_t slot, uint32_t space, const void * const byte *data = (const byte *)ByteCode; // just for convenience + if(ByteCode == NULL || BytecodeLength == 0) + return false; + if(header->fourcc != FOURCC_DXBC) return false; @@ -978,6 +984,9 @@ bool DXBCContainer::CheckForDebugInfo(const void *ByteCode, size_t ByteCodeLengt char *data = (char *)ByteCode; // just for convenience + if(ByteCode == NULL || ByteCodeLength == 0) + return false; + if(header->fourcc != FOURCC_DXBC) return false; @@ -1003,6 +1012,9 @@ bool DXBCContainer::CheckForDXIL(const void *ByteCode, size_t ByteCodeLength) char *data = (char *)ByteCode; // just for convenience + if(ByteCode == NULL || ByteCodeLength == 0) + return false; + if(header->fourcc != FOURCC_DXBC) return false; @@ -1029,6 +1041,9 @@ rdcstr DXBCContainer::GetDebugBinaryPath(const void *ByteCode, size_t ByteCodeLe char *data = (char *)ByteCode; // just for convenience + if(ByteCode == NULL || ByteCodeLength == 0) + return debugPath; + if(header->fourcc != FOURCC_DXBC) return debugPath;