From 3ce8d68fbadbed6a3826b045bc1f62e6bc384d10 Mon Sep 17 00:00:00 2001 From: baldurk Date: Thu, 2 Sep 2021 14:22:46 +0100 Subject: [PATCH] Emit metadata block --- .../driver/shaders/dxbc/dxbc_container.cpp | 39 ++++- .../driver/shaders/dxbc/dxbc_container.h | 1 + .../driver/shaders/dxil/dxil_bytecode.cpp | 3 +- renderdoc/driver/shaders/dxil/dxil_bytecode.h | 2 +- .../shaders/dxil/dxil_bytecode_editor.cpp | 136 ++++++++++++++++-- .../driver/shaders/dxil/dxil_debuginfo.cpp | 6 +- .../driver/shaders/dxil/llvm_encoder.cpp | 112 +++++++++++---- renderdoc/driver/shaders/dxil/llvm_encoder.h | 29 ++-- 8 files changed, 267 insertions(+), 61 deletions(-) diff --git a/renderdoc/driver/shaders/dxbc/dxbc_container.cpp b/renderdoc/driver/shaders/dxbc/dxbc_container.cpp index faf5bf9dc..e67ac6878 100644 --- a/renderdoc/driver/shaders/dxbc/dxbc_container.cpp +++ b/renderdoc/driver/shaders/dxbc/dxbc_container.cpp @@ -714,7 +714,7 @@ void DXBCContainer::ReplaceDXBCBytecode(bytebuf &ByteCode, const rdcarraynumChunks; c++) - chunkOffsets[c + 1] = chunkOffsets[c]; + chunkOffsets[c] = chunkOffsets[c + 1] - size; header->numChunks--; + header->fileLength -= size; + + ByteCode.erase(offs, size); + + break; } - else if(*fourcc == FOURCC_DXIL) + } + + HashContainer(ByteCode.data(), ByteCode.size()); +} + +void DXBCContainer::ReplaceDXILBytecode(bytebuf &ByteCode, const bytebuf &replacement) +{ + FileHeader *header = (FileHeader *)ByteCode.data(); + + if(header->fourcc != FOURCC_DXBC) + return; + + if(header->fileLength != (uint32_t)ByteCode.size()) + return; + + uint32_t *chunkOffsets = + (uint32_t *)(ByteCode.data() + sizeof(FileHeader)); // right after the header + + for(uint32_t chunkIdx = 0; chunkIdx < header->numChunks; chunkIdx++) + { + uint32_t offs = chunkOffsets[chunkIdx]; + + uint32_t *fourcc = (uint32_t *)(ByteCode.data() + offs); + uint32_t *chunkSize = (uint32_t *)(fourcc + 1); + + if(*fourcc == FOURCC_DXIL) { int32_t diff = int32_t(replacement.size()) - int32_t(*chunkSize); diff --git a/renderdoc/driver/shaders/dxbc/dxbc_container.h b/renderdoc/driver/shaders/dxbc/dxbc_container.h index bd24109d7..982dac902 100644 --- a/renderdoc/driver/shaders/dxbc/dxbc_container.h +++ b/renderdoc/driver/shaders/dxbc/dxbc_container.h @@ -164,6 +164,7 @@ public: void FillStateInstructionInfo(ShaderDebugState &state) const; static void ReplaceDXBCBytecode(bytebuf &ByteCode, const rdcarray &replacement); + static void StripDXILDebugInfo(bytebuf &ByteCode); static void ReplaceDXILBytecode(bytebuf &ByteCode, const bytebuf &replacement); const DXBCBytecode::Program *GetDXBCByteCode() const { return m_DXBCByteCode; } diff --git a/renderdoc/driver/shaders/dxil/dxil_bytecode.cpp b/renderdoc/driver/shaders/dxil/dxil_bytecode.cpp index affe5ead6..268a525c9 100644 --- a/renderdoc/driver/shaders/dxil/dxil_bytecode.cpp +++ b/renderdoc/driver/shaders/dxil/dxil_bytecode.cpp @@ -920,7 +920,7 @@ Program::Program(const byte *bytes, size_t length) else if(IS_KNOWN(metaRecord.id, MetaDataRecord::KIND)) { size_t kind = (size_t)metaRecord.ops[0]; - m_Kinds.resize(RDCMAX(m_Kinds.size(), kind + 1)); + m_Kinds.resize_for_index(kind); m_Kinds[kind] = metaRecord.getString(1); continue; } @@ -939,6 +939,7 @@ Program::Program(const byte *bytes, size_t length) if(IS_KNOWN(metaRecord.id, MetaDataRecord::STRING_OLD)) { meta.isConstant = true; + meta.isString = true; meta.str = metaRecord.getString(); } else if(IS_KNOWN(metaRecord.id, MetaDataRecord::VALUE)) diff --git a/renderdoc/driver/shaders/dxil/dxil_bytecode.h b/renderdoc/driver/shaders/dxil/dxil_bytecode.h index d0b22eb6b..0bd39256a 100644 --- a/renderdoc/driver/shaders/dxil/dxil_bytecode.h +++ b/renderdoc/driver/shaders/dxil/dxil_bytecode.h @@ -480,7 +480,7 @@ struct Metadata ~Metadata(); uint32_t id = ~0U; - bool isDistinct = false, isConstant = false; + bool isDistinct = false, isConstant = false, isString = false; const Constant *constant = NULL; diff --git a/renderdoc/driver/shaders/dxil/dxil_bytecode_editor.cpp b/renderdoc/driver/shaders/dxil/dxil_bytecode_editor.cpp index 89c9a672d..55e4034c2 100644 --- a/renderdoc/driver/shaders/dxil/dxil_bytecode_editor.cpp +++ b/renderdoc/driver/shaders/dxil/dxil_bytecode_editor.cpp @@ -49,23 +49,43 @@ bytebuf DXIL::ProgramEditor::EncodeProgram() const LLVMBC::BitcodeWriter writer(ret); - uint64_t maxAlign = 0; - uint32_t maxGlobalType = 0; + LLVMBC::BitcodeWriter::Config cfg = {}; - auto getTypeID = [this](const Type *t) { return uint64_t(t - m_Types.begin()); }; +#define getTypeID(t) uint64_t(t - m_Types.begin()) +#define getConstantID(c) uint64_t(c - m_Constants.begin()) +#define getMetaID(m) uint64_t(m - m_Metadata.begin()) +#define getMetaIDOrNull(m) (m ? (uint64_t(m - m_Metadata.begin()) + 1) : 0) for(size_t i = 0; i < m_GlobalVars.size(); i++) { - maxAlign = RDCMAX(m_GlobalVars[i].align, maxAlign); + cfg.maxAlign = RDCMAX(m_GlobalVars[i].align, cfg.maxAlign); RDCASSERT(m_GlobalVars[i].type->type == Type::Pointer); uint32_t typeIndex = uint32_t(getTypeID(m_GlobalVars[i].type->inner)); - maxGlobalType = RDCMAX(typeIndex, maxGlobalType); + cfg.maxGlobalType = RDCMAX(typeIndex, cfg.maxGlobalType); } for(size_t i = 0; i < m_Functions.size(); i++) - maxAlign = RDCMAX(m_Functions[i].align, maxAlign); + cfg.maxAlign = RDCMAX(m_Functions[i].align, cfg.maxAlign); - size_t numGlobalConsts = 0; + for(size_t i = 0; i < m_Metadata.size(); i++) + { + if(m_Metadata[i].isString) + cfg.hasMetaString = true; + + if(m_Metadata[i].debugLoc) + cfg.hasDebugLoc = true; + } + + for(size_t i = 0; i < m_NamedMeta.size(); i++) + { + if(m_NamedMeta[i].isString) + cfg.hasMetaString = true; + + if(m_NamedMeta[i].debugLoc) + cfg.hasDebugLoc = true; + } + + cfg.hasNamedMeta = !m_NamedMeta.empty(); for(size_t i = m_GlobalVars.size() + m_Functions.size(); i < m_Symbols.size(); i++) { @@ -73,10 +93,13 @@ bytebuf DXIL::ProgramEditor::EncodeProgram() const if(m_Symbols[i].type != SymbolType::Constant) break; - numGlobalConsts++; + cfg.numGlobalConsts++; } - writer.ConfigureSizes(m_Types.size(), numGlobalConsts, m_Sections.size(), maxAlign, maxGlobalType); + cfg.numTypes = m_Types.size(); + cfg.numSections = m_Sections.size(); + + writer.ConfigureSizes(cfg); writer.BeginBlock(LLVMBC::KnownBlock::MODULE_BLOCK); @@ -442,7 +465,7 @@ bytebuf DXIL::ProgramEditor::EncodeProgram() const for(size_t m = 0; m < c.members.size(); m++) { vals.push_back(getTypeID(c.members[m]->type)); - vals.push_back(c.members[m] - m_Constants.begin()); + vals.push_back(getConstantID(c.members[m])); } writer.Record(LLVMBC::ConstantsRecord::EVAL_GEP, vals); @@ -452,9 +475,9 @@ bytebuf DXIL::ProgramEditor::EncodeProgram() const uint64_t cast = EncodeCast(c.op); RDCASSERT(cast != ~0U); - writer.Record(LLVMBC::ConstantsRecord::EVAL_CAST, - {EncodeCast(c.op), getTypeID(c.type), getTypeID(c.inner->type), - uint64_t(c.inner - m_Constants.begin())}); + writer.Record( + LLVMBC::ConstantsRecord::EVAL_CAST, + {EncodeCast(c.op), getTypeID(c.type), getTypeID(c.inner->type), getConstantID(c.inner)}); } else if(c.type->scalarType == Type::Int) { @@ -501,7 +524,7 @@ bytebuf DXIL::ProgramEditor::EncodeProgram() const vals.reserve(c.members.size()); for(size_t m = 0; m < c.members.size(); m++) - vals.push_back(c.members[m] - m_Constants.begin()); + vals.push_back(getConstantID(c.members[m])); writer.Record(LLVMBC::ConstantsRecord::AGGREGATE, vals); } @@ -511,6 +534,91 @@ bytebuf DXIL::ProgramEditor::EncodeProgram() const writer.EndBlock(); } + if(!m_Metadata.empty()) + { + writer.BeginBlock(LLVMBC::KnownBlock::METADATA_BLOCK); + + writer.EmitMetaDataAbbrev(); + + rdcarray vals; + + bool errored = false; + + for(size_t i = 0; i < m_Metadata.size(); i++) + { + if(m_Metadata[i].isString) + { + writer.Record(LLVMBC::MetaDataRecord::STRING_OLD, m_Metadata[i].str); + } + else if(m_Metadata[i].isConstant) + { + writer.Record(LLVMBC::MetaDataRecord::VALUE, + {getTypeID(m_Metadata[i].type), getConstantID(m_Metadata[i].constant)}); + } + else if(m_Metadata[i].dwarf || m_Metadata[i].debugLoc) + { + if(!errored) + RDCERR("Unexpected debug metadata node - expect to only encode stripped DXIL chunks"); + errored = true; + + // replace this with the first NULL constant value + for(size_t c = 0; c < m_Constants.size(); c++) + { + if(m_Constants[c].nullconst) + { + writer.Record(LLVMBC::MetaDataRecord::VALUE, + {getTypeID(m_Constants[c].type), (uint64_t)c}); + } + } + } + else + { + vals.clear(); + for(size_t m = 0; m < m_Metadata[i].children.size(); m++) + vals.push_back(getMetaIDOrNull(m_Metadata[i].children[m])); + + writer.Record(m_Metadata[i].isDistinct ? LLVMBC::MetaDataRecord::DISTINCT_NODE + : LLVMBC::MetaDataRecord::NODE, + vals); + } + } + + for(size_t i = 0; i < m_NamedMeta.size(); i++) + { + writer.Record(LLVMBC::MetaDataRecord::NAME, m_NamedMeta[i].name); + + vals.clear(); + for(size_t m = 0; m < m_NamedMeta[i].children.size(); m++) + vals.push_back(getMetaID(m_NamedMeta[i].children[m])); + + writer.Record(LLVMBC::MetaDataRecord::NAMED_NODE, vals); + } + + writer.EndBlock(); + } + + if(!m_Kinds.empty()) + { + writer.BeginBlock(LLVMBC::KnownBlock::METADATA_BLOCK); + + rdcarray vals; + + for(size_t i = 0; i < m_Kinds.size(); i++) + { + if(m_Kinds[i].empty()) + continue; + + vals.clear(); + vals.push_back(i); + for(char c : m_Kinds[i]) + vals.push_back(c); + + writer.Record(LLVMBC::MetaDataRecord::KIND, vals); + } + + writer.EndBlock(); + } + writer.EndBlock(); ProgramHeader header; diff --git a/renderdoc/driver/shaders/dxil/dxil_debuginfo.cpp b/renderdoc/driver/shaders/dxil/dxil_debuginfo.cpp index db8772cff..4759a4f16 100644 --- a/renderdoc/driver/shaders/dxil/dxil_debuginfo.cpp +++ b/renderdoc/driver/shaders/dxil/dxil_debuginfo.cpp @@ -33,9 +33,9 @@ bool Program::ParseDebugMetaRecord(const LLVMBC::BlockOrRecord &metaRecord, Meta { LLVMBC::MetaDataRecord id = (LLVMBC::MetaDataRecord)metaRecord.id; - auto getNonNullMeta = [this](uint64_t id) { return &m_Metadata[size_t(id)]; }; - auto getMeta = [this](uint64_t id) { return id ? &m_Metadata[size_t(id - 1)] : NULL; }; - auto getMetaString = [this](uint64_t id) { return id ? &m_Metadata[size_t(id - 1)].str : NULL; }; +#define getNonNullMeta(id) &m_Metadata[size_t(id)] +#define getMeta(id) (id ? &m_Metadata[size_t(id - 1)] : NULL) +#define getMetaString(id) (id ? &m_Metadata[size_t(id - 1)].str : NULL) if(id == LLVMBC::MetaDataRecord::FILE) { diff --git a/renderdoc/driver/shaders/dxil/llvm_encoder.cpp b/renderdoc/driver/shaders/dxil/llvm_encoder.cpp index a757c1b55..b2ba2f30b 100644 --- a/renderdoc/driver/shaders/dxil/llvm_encoder.cpp +++ b/renderdoc/driver/shaders/dxil/llvm_encoder.cpp @@ -250,6 +250,28 @@ AbbrevDefinition TypeAbbrevDefs[] = { }, }; +enum class MetadataAbbrev +{ + String, + DebugLocation, + Name, +}; + +AbbrevDefinition MetadataAbbrevDefs[] = { + // String + { + AbbLiteral(MetaDataRecord::STRING_OLD), AbbArray(), AbbFixed(8), + }, + // DebugLocation + { + AbbLiteral(MetaDataRecord::LOCATION), AbbFixed(1), AbbVBR(6), AbbVBR(8), AbbVBR(6), AbbVBR(6), + }, + // Name + { + AbbLiteral(MetaDataRecord::NAME), AbbArray(), AbbFixed(8), + }, +}; + static AbbrevDefinition *GetAbbrevDefs(KnownBlock block) { AbbrevDefinition *ret = NULL; @@ -260,6 +282,7 @@ static AbbrevDefinition *GetAbbrevDefs(KnownBlock block) case KnownBlock::CONSTANTS_BLOCK: ret = ConstantsAbbrevDefs; break; case KnownBlock::FUNCTION_BLOCK: ret = FunctionAbbrevDefs; break; case KnownBlock::TYPE_BLOCK: ret = TypeAbbrevDefs; break; + case KnownBlock::METADATA_BLOCK: ret = MetadataAbbrevDefs; break; default: break; } @@ -276,6 +299,7 @@ static uint32_t GetNumAbbrevDefs(KnownBlock block) case KnownBlock::CONSTANTS_BLOCK: ret = ARRAY_COUNT(ConstantsAbbrevDefs); break; case KnownBlock::FUNCTION_BLOCK: ret = ARRAY_COUNT(FunctionAbbrevDefs); break; case KnownBlock::TYPE_BLOCK: ret = ARRAY_COUNT(TypeAbbrevDefs); break; + case KnownBlock::METADATA_BLOCK: ret = ARRAY_COUNT(MetadataAbbrevDefs); break; default: break; } @@ -288,8 +312,6 @@ BitcodeWriter::BitcodeWriter(bytebuf &buf) : b(buf) curBlock = KnownBlock::Count; abbrevSize = 2; - - m_GlobalVarAbbrev = ~0U; } BitcodeWriter::~BitcodeWriter() @@ -346,6 +368,11 @@ void BitcodeWriter::BeginBlock(KnownBlock block) break; } + case KnownBlock::METADATA_BLOCK: + { + // this is handled manually, only in the first global metadata block + break; + } default: { uint32_t numAbbrevDefs = GetNumAbbrevDefs(block); @@ -400,9 +427,9 @@ void BitcodeWriter::WriteAbbrevDefinition(AbbrevParam *abbrev) AbbrevParam param = abbrev[p]; if(param.value == MagicFixedSizeNumTypes) - param.value = m_NumTypeBits; + param.value = m_Cfg.numTypes; if(param.value == MagicFixedSizeNumConstants) - param.value = m_NumConstantBits; + param.value = m_Cfg.numGlobalConsts; const bool lit = param.encoding == AbbrevEncoding::Literal; b.fixed(1, lit); @@ -419,27 +446,22 @@ void BitcodeWriter::WriteAbbrevDefinition(AbbrevParam *abbrev) } } -void BitcodeWriter::ConfigureSizes(size_t numTypes, size_t numGlobalConsts, size_t numSections, - uint64_t maxAlign, uint32_t maxGlobalType) +void BitcodeWriter::ConfigureSizes(Config cfg) { - m_NumTypeBits = 32 - Bits::CountLeadingZeroes((uint32_t)numTypes); - m_NumConstantBits = 32 - Bits::CountLeadingZeroes((uint32_t)numGlobalConsts); + m_Cfg = cfg; - m_GlobalTypeBits = 32 - Bits::CountLeadingZeroes(maxGlobalType); + m_Cfg.numTypes = 32 - Bits::CountLeadingZeroes((uint32_t)m_Cfg.numTypes); + m_Cfg.numGlobalConsts = 32 - Bits::CountLeadingZeroes((uint32_t)m_Cfg.numGlobalConsts); - if(numSections == 0) - m_NumSectionBits = 0; - else - m_NumSectionBits = 32 - Bits::CountLeadingZeroes((uint32_t)numSections); + m_Cfg.maxGlobalType = 32 - Bits::CountLeadingZeroes(m_Cfg.maxGlobalType); - if(maxAlign == 0) + if(m_Cfg.numSections > 0) + m_Cfg.numSections = 32 - Bits::CountLeadingZeroes((uint32_t)m_Cfg.numSections); + + if(m_Cfg.maxAlign > 0) { - m_AlignBits = 0; - } - else - { - uint32_t encodedAlign = 32 - Bits::CountLeadingZeroes((uint32_t)maxAlign); - m_AlignBits = 32 - Bits::CountLeadingZeroes(encodedAlign); + uint32_t encodedAlign = 32 - Bits::CountLeadingZeroes((uint32_t)cfg.maxAlign); + m_Cfg.maxAlign = 32 - Bits::CountLeadingZeroes(encodedAlign); } } @@ -468,16 +490,16 @@ void BitcodeWriter::EmitGlobalVarAbbrev() { m_GlobalVarAbbrev = (uint32_t)curAbbrevs.size(); - AbbrevParam align = AbbFixed(m_AlignBits); - if(m_AlignBits == 0) + AbbrevParam align = AbbFixed(m_Cfg.maxAlign); + if(m_Cfg.maxAlign == 0) align = AbbLiteral(0); - AbbrevParam section = AbbFixed(m_NumSectionBits); - if(m_NumSectionBits == 0) + AbbrevParam section = AbbFixed(m_Cfg.numSections); + if(m_Cfg.numSections == 0) section = AbbLiteral(0); m_GlobalVarAbbrevDef[0] = AbbLiteral(ModuleRecord::GLOBALVAR); - m_GlobalVarAbbrevDef[1] = AbbFixed(m_GlobalTypeBits); + m_GlobalVarAbbrevDef[1] = AbbFixed(m_Cfg.maxGlobalType); m_GlobalVarAbbrevDef[2] = AbbVBR(6); m_GlobalVarAbbrevDef[3] = AbbVBR(6); m_GlobalVarAbbrevDef[4] = AbbFixed(5); @@ -487,6 +509,28 @@ void BitcodeWriter::EmitGlobalVarAbbrev() WriteAbbrevDefinition(m_GlobalVarAbbrevDef); } +void BitcodeWriter::EmitMetaDataAbbrev() +{ + // metadata only emits its abbrev if there are the relevant nodes + if(m_Cfg.hasMetaString) + { + m_MetaStringAbbrev = (uint32_t)curAbbrevs.size(); + WriteAbbrevDefinition(MetadataAbbrevDefs[(uint32_t)MetadataAbbrev::String]); + } + if(m_Cfg.hasDebugLoc) + { + m_MetaLocationAbbrev = (uint32_t)curAbbrevs.size(); + WriteAbbrevDefinition(MetadataAbbrevDefs[(uint32_t)MetadataAbbrev::DebugLocation]); + } + if(m_Cfg.hasNamedMeta) + { + m_MetaNameAbbrev = (uint32_t)curAbbrevs.size(); + WriteAbbrevDefinition(MetadataAbbrevDefs[(uint32_t)MetadataAbbrev::Name]); + } + + // we don't handle GENERIC_DEBUG +} + uint32_t BitcodeWriter::GetAbbrevID(uint32_t id) { // the id is a block-local index, starting from 0, of the abbrevs defined for that block. @@ -664,6 +708,14 @@ void BitcodeWriter::AutoRecord(uint32_t record, const rdcarray &vals) default: break; } break; + case KnownBlock::METADATA_BLOCK: + switch(MetaDataRecord(record)) + { + case MetaDataRecord::STRING_OLD: idx = (uint32_t)m_MetaStringAbbrev; break; + case MetaDataRecord::LOCATION: idx = (uint32_t)m_MetaLocationAbbrev; break; + case MetaDataRecord::NAME: idx = (uint32_t)m_MetaNameAbbrev; break; + default: break; + } default: break; } @@ -697,8 +749,6 @@ void BitcodeWriter::Abbrev(AbbrevParam *abbr, uint32_t record, const rdcarrayencoding != AbbrevEncoding::Unknown) { - RDCASSERT(i < vals.size()); - // only one array per abbrev, consume the rest of the vals if(abbr->encoding == AbbrevEncoding::Array) { @@ -715,6 +765,8 @@ void BitcodeWriter::Abbrev(AbbrevParam *abbr, uint32_t record, const rdcarrayencoding == AbbrevEncoding::Blob) { + RDCASSERT(i < vals.size()); + // expect vals to be length then blob pointer packed into uint64_t size_t length = (size_t)vals[i]; byte *blob = (byte *)vals[i + 1]; @@ -722,6 +774,8 @@ void BitcodeWriter::Abbrev(AbbrevParam *abbr, uint32_t record, const rdcarray &vals); @@ -89,17 +102,17 @@ private: BitWriter b; - uint32_t m_NumTypeBits; - uint32_t m_NumConstantBits; - uint32_t m_GlobalTypeBits; - uint32_t m_NumSectionBits; - uint32_t m_AlignBits; + Config m_Cfg; size_t abbrevSize; rdcarray curAbbrevs; KnownBlock curBlock; - uint32_t m_GlobalVarAbbrev; + uint32_t m_GlobalVarAbbrev = ~0U; + uint32_t m_MetaStringAbbrev = ~0U; + uint32_t m_MetaLocationAbbrev = ~0U; + uint32_t m_MetaNameAbbrev = ~0U; + AbbrevParam m_GlobalVarAbbrevDef[10] = {}; struct BlockContext