From 0cb940fcbe8ae551e9955a109dab2d100b8ec4a2 Mon Sep 17 00:00:00 2001 From: baldurk Date: Mon, 30 May 2016 12:12:18 +0200 Subject: [PATCH] Support LZ4 compression of stripped separate debug blobs --- docs/getting_started/faq.rst | 6 ++- docs/getting_started/tips_tricks.rst | 4 +- renderdoc/driver/d3d11/d3d11_resources.cpp | 54 ++++++++++++++++++++-- 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/docs/getting_started/faq.rst b/docs/getting_started/faq.rst index c2a68f519..10cb74596 100644 --- a/docs/getting_started/faq.rst +++ b/docs/getting_started/faq.rst @@ -208,4 +208,8 @@ This optional information is generated by the compiler, but is not required for The simplest solution is just to avoid stripping the data when using RenderDoc, but that isn't always possible. Instead RenderDoc allows you to use API-specific methods to specify where the unstripped data can be found. This means you can save the unstripped shader to a debug location and then either store this location with the shader, or specify it at runtime. On replay RenderDoc will expect the data to be available at that location and it will load it up instead. -For details on this method, check out :doc:`tips_tricks`. +The path you specify (with the stripped shader, or at runtime) can be either absolute or relative. If it's relative, you must configure a shader search path in the :doc:`../window/options_window`. + +The stripped shader file stored on disk can also be compressed with LZ4 to save space as often most of the size is made up for shader source text which compresses well. To do this, simply compress the contents of the file and prepend the pathname (either absolute or relative, specified in the shader blob or at runtime) with ``lz4#``. + +For example code using this method, check out :doc:`tips_tricks`. diff --git a/docs/getting_started/tips_tricks.rst b/docs/getting_started/tips_tricks.rst index d274972a1..ad206cead 100644 --- a/docs/getting_started/tips_tricks.rst +++ b/docs/getting_started/tips_tricks.rst @@ -78,7 +78,9 @@ This page is a random hodge-podge of different tips and tricks that might not be GUID RENDERDOC_ShaderDebugMagicValue = RENDERDOC_ShaderDebugMagicValue_value; // GUID value in renderdoc_app.h ID3D11VertexShader *shader = ...; - std::string pathName = ...; // path name is in UTF-8 + std::string pathName = "/path/to/saved/blob"; // path name is in UTF-8 + // path name can also be prefixed with lz4# to indicate the blob is compressed + pathName = "lz4#/path/to/saved/blob"; // string parameter must be NULL-terminated, and in UTF-8 shader->SetPrivateData(RENDERDOC_ShaderDebugMagicValue, (UINT)pathName.length(), pathName.c_str()); diff --git a/renderdoc/driver/d3d11/d3d11_resources.cpp b/renderdoc/driver/d3d11/d3d11_resources.cpp index b3cd89d27..9250cfdf2 100644 --- a/renderdoc/driver/d3d11/d3d11_resources.cpp +++ b/renderdoc/driver/d3d11/d3d11_resources.cpp @@ -24,6 +24,7 @@ ******************************************************************************/ #include "driver/d3d11/d3d11_resources.h" +#include "3rdparty/lz4/lz4.h" #include "api/app/renderdoc_app.h" #include "driver/d3d11/d3d11_context.h" #include "driver/dxgi/dxgi_wrapped.h" @@ -79,10 +80,21 @@ void WrappedShader::ShaderEntry::TryReplaceOriginalByteCode() if(!originalPath.empty()) { + bool lz4 = false; + + if(!strncmp(originalPath.c_str(), "lz4#", 4)) + { + originalPath = originalPath.substr(4); + lz4 = true; + } + // could support more if we're willing to compile in the decompressor + FILE *originalShaderFile = NULL; size_t numSearchPaths = m_DebugInfoSearchPaths ? m_DebugInfoSearchPaths->size() : 0; + string foundPath; + // while we haven't found a file, keep trying through the search paths. For i==0 // check the path on its own, in case it's an absolute path. for(size_t i = 0; originalShaderFile == NULL && i <= numSearchPaths; i++) @@ -90,12 +102,14 @@ void WrappedShader::ShaderEntry::TryReplaceOriginalByteCode() if(i == 0) { originalShaderFile = FileIO::fopen(originalPath.c_str(), "rb"); + foundPath = originalPath; continue; } else { - std::string &searchPath = (*m_DebugInfoSearchPaths)[i - 1]; - originalShaderFile = FileIO::fopen((searchPath + "/" + originalPath).c_str(), "rb"); + const std::string &searchPath = (*m_DebugInfoSearchPaths)[i - 1]; + foundPath = searchPath + "/" + originalPath; + originalShaderFile = FileIO::fopen(foundPath.c_str(), "rb"); } } @@ -106,7 +120,7 @@ void WrappedShader::ShaderEntry::TryReplaceOriginalByteCode() uint64_t originalShaderSize = FileIO::ftell64(originalShaderFile); FileIO::fseek64(originalShaderFile, 0, SEEK_SET); - if(originalShaderSize >= m_Bytecode.size()) + if(lz4 || originalShaderSize >= m_Bytecode.size()) { vector originalBytecode; @@ -114,6 +128,40 @@ void WrappedShader::ShaderEntry::TryReplaceOriginalByteCode() FileIO::fread(&originalBytecode[0], sizeof(byte), (size_t)originalShaderSize, originalShaderFile); + if(lz4) + { + vector decompressed; + + // first try decompressing to 1MB flat + decompressed.resize(100 * 1024); + + int ret = LZ4_decompress_safe((const char *)&originalBytecode[0], (char *)&decompressed[0], + (int)originalBytecode.size(), (int)decompressed.size()); + + if(ret < 0) + { + // if it failed, either source is corrupt or we didn't allocate enough space. + // Just allocate 255x compressed size since it can't need any more than that. + decompressed.resize(255 * originalBytecode.size()); + + ret = LZ4_decompress_safe((const char *)&originalBytecode[0], (char *)&decompressed[0], + (int)originalBytecode.size(), (int)decompressed.size()); + + if(ret < 0) + { + RDCERR("Failed to decompress LZ4 data from %s", foundPath.c_str()); + return; + } + } + + RDCASSERT(ret > 0, ret); + + // we resize and memcpy instead of just doing .swap() because that would + // transfer over the over-large pessimistic capacity needed for decompression + originalBytecode.resize(ret); + memcpy(&originalBytecode[0], &decompressed[0], originalBytecode.size()); + } + if(DXBC::DXBCFile::CheckForDebugInfo((const void *)&originalBytecode[0], originalBytecode.size())) {