Support LZ4 compression of stripped separate debug blobs

docs/getting_started/faq.rst

@@ -208,4 +208,8 @@ This optional information is generated by the compiler, but is not required for
 
 The simplest solution is just to avoid stripping the data when using RenderDoc, but that isn't always possible. Instead RenderDoc allows you to use API-specific methods to specify where the unstripped data can be found. This means you can save the unstripped shader to a debug location and then either store this location with the shader, or specify it at runtime. On replay RenderDoc will expect the data to be available at that location and it will load it up instead.
 
-For details on this method, check out :doc:`tips_tricks`.
+The path you specify (with the stripped shader, or at runtime) can be either absolute or relative. If it's relative, you must configure a shader search path in the :doc:`../window/options_window`.
+
+The stripped shader file stored on disk can also be compressed with LZ4 to save space as often most of the size is made up for shader source text which compresses well. To do this, simply compress the contents of the file and prepend the pathname (either absolute or relative, specified in the shader blob or at runtime) with ``lz4#``.
+
+For example code using this method, check out :doc:`tips_tricks`.

docs/getting_started/tips_tricks.rst

@@ -78,7 +78,9 @@ This page is a random hodge-podge of different tips and tricks that might not be
        GUID RENDERDOC_ShaderDebugMagicValue = RENDERDOC_ShaderDebugMagicValue_value; // GUID value in renderdoc_app.h
 
        ID3D11VertexShader *shader = ...;
-       std::string pathName = ...; // path name is in UTF-8
+       std::string pathName = "/path/to/saved/blob"; // path name is in UTF-8
+       // path name can also be prefixed with lz4# to indicate the blob is compressed
+       pathName = "lz4#/path/to/saved/blob";
 
        // string parameter must be NULL-terminated, and in UTF-8
        shader->SetPrivateData(RENDERDOC_ShaderDebugMagicValue, (UINT)pathName.length(), pathName.c_str());

renderdoc/driver/d3d11/d3d11_resources.cpp

@@ -24,6 +24,7 @@
  ******************************************************************************/
 
 #include "driver/d3d11/d3d11_resources.h"
+#include "3rdparty/lz4/lz4.h"
 #include "api/app/renderdoc_app.h"
 #include "driver/d3d11/d3d11_context.h"
 #include "driver/dxgi/dxgi_wrapped.h"
@@ -79,10 +80,21 @@ void WrappedShader::ShaderEntry::TryReplaceOriginalByteCode()
 
     if(!originalPath.empty())
     {
+      bool lz4 = false;
+
+      if(!strncmp(originalPath.c_str(), "lz4#", 4))
+      {
+        originalPath = originalPath.substr(4);
+        lz4 = true;
+      }
+      // could support more if we're willing to compile in the decompressor
+
       FILE *originalShaderFile = NULL;
 
       size_t numSearchPaths = m_DebugInfoSearchPaths ? m_DebugInfoSearchPaths->size() : 0;
 
+      string foundPath;
+
       // while we haven't found a file, keep trying through the search paths. For i==0
       // check the path on its own, in case it's an absolute path.
       for(size_t i = 0; originalShaderFile == NULL && i <= numSearchPaths; i++)
@@ -90,12 +102,14 @@ void WrappedShader::ShaderEntry::TryReplaceOriginalByteCode()
         if(i == 0)
         {
           originalShaderFile = FileIO::fopen(originalPath.c_str(), "rb");
+          foundPath = originalPath;
           continue;
         }
         else
         {
-          std::string &searchPath = (*m_DebugInfoSearchPaths)[i - 1];
-          originalShaderFile = FileIO::fopen((searchPath + "/" + originalPath).c_str(), "rb");
+          const std::string &searchPath = (*m_DebugInfoSearchPaths)[i - 1];
+          foundPath = searchPath + "/" + originalPath;
+          originalShaderFile = FileIO::fopen(foundPath.c_str(), "rb");
         }
       }
 
@@ -106,7 +120,7 @@ void WrappedShader::ShaderEntry::TryReplaceOriginalByteCode()
       uint64_t originalShaderSize = FileIO::ftell64(originalShaderFile);
       FileIO::fseek64(originalShaderFile, 0, SEEK_SET);
 
-      if(originalShaderSize >= m_Bytecode.size())
+      if(lz4 || originalShaderSize >= m_Bytecode.size())
       {
         vector<byte> originalBytecode;
 
@@ -114,6 +128,40 @@ void WrappedShader::ShaderEntry::TryReplaceOriginalByteCode()
         FileIO::fread(&originalBytecode[0], sizeof(byte), (size_t)originalShaderSize,
                       originalShaderFile);
 
+        if(lz4)
+        {
+          vector<byte> decompressed;
+
+          // first try decompressing to 1MB flat
+          decompressed.resize(100 * 1024);
+
+          int ret = LZ4_decompress_safe((const char *)&originalBytecode[0], (char *)&decompressed[0],
+                                        (int)originalBytecode.size(), (int)decompressed.size());
+
+          if(ret < 0)
+          {
+            // if it failed, either source is corrupt or we didn't allocate enough space.
+            // Just allocate 255x compressed size since it can't need any more than that.
+            decompressed.resize(255 * originalBytecode.size());
+
+            ret = LZ4_decompress_safe((const char *)&originalBytecode[0], (char *)&decompressed[0],
+                                      (int)originalBytecode.size(), (int)decompressed.size());
+
+            if(ret < 0)
+            {
+              RDCERR("Failed to decompress LZ4 data from %s", foundPath.c_str());
+              return;
+            }
+          }
+
+          RDCASSERT(ret > 0, ret);
+
+          // we resize and memcpy instead of just doing .swap() because that would
+          // transfer over the over-large pessimistic capacity needed for decompression
+          originalBytecode.resize(ret);
+          memcpy(&originalBytecode[0], &decompressed[0], originalBytecode.size());
+        }
+
         if(DXBC::DXBCFile::CheckForDebugInfo((const void *)&originalBytecode[0],
                                              originalBytecode.size()))
         {