RenderDoc Core APIs to support external file dependencies in captures

renderdoc/api/replay/renderdoc_tostr.inl

@@ -1128,6 +1128,7 @@ rdcstr DoStringise(const SectionType &el)
     STRINGISE_ENUM_CLASS_NAMED(EditedShaders, "renderdoc/ui/edits");
     STRINGISE_ENUM_CLASS_NAMED(D3D12Core, "renderdoc/internal/d3d12core");
     STRINGISE_ENUM_CLASS_NAMED(D3D12SDKLayers, "renderdoc/internal/d3d12sdklayers");
+    STRINGISE_ENUM_CLASS_NAMED(EmbeddedExternalFiles, "renderdoc/internal/embeddedexternalfiles");
   }
   END_ENUM_STRINGISE();
 }

renderdoc/api/replay/replay_enums.h

@@ -110,6 +110,12 @@ version of RenderDoc that addes a new section type. They should be considered eq
   This section contains an internal copy of D3D12SDKLayers for replaying.
 
   The name for this section will be "renderdoc/internal/d3d12sdklayers".
+
+.. data:: EmbeddedExternalFiles
+
+  This section contains externally referenced files that have been embedded into the capture.
+
+  The name for this section will be "renderdoc/internal/embeddedexternalfiles".
 )");
 enum class SectionType : uint32_t
 {
@@ -126,6 +132,7 @@ enum class SectionType : uint32_t
   EditedShaders,
   D3D12Core,
   D3D12SDKLayers,
+  EmbeddedExternalFiles,
   Count,
 };
 

renderdoc/core/core.cpp

@@ -419,6 +419,8 @@ RenderDoc::RenderDoc()
 
   m_TargetControlThreadShutdown = false;
   m_ControlClientThreadShutdown = false;
+
+  ClearTrackedFiles();
 }
 
 void RenderDoc::Initialise()
@@ -532,6 +534,8 @@ void RenderDoc::Initialise()
 
   m_ExHandler = NULL;
 
+  ClearTrackedFiles();
+
   RecreateCrashHandler();
 
   // begin printing to stdout/stderr after this point, earlier logging is debugging
@@ -2238,6 +2242,305 @@ bool RenderDoc::HasActiveFrameCapturer(RDCDriver driver)
   return false;
 }
 
+bool RenderDoc::GetTrackedFileData(const rdcstr &nickname, bytebuf &data) const
+{
+  SCOPED_READLOCK(m_TrackedFilesLock);
+  for(const TrackedFile &f : m_TrackedFiles)
+  {
+    if(f.nickname == nickname)
+    {
+      data = f.data;
+      return true;
+    }
+  }
+  return false;
+}
+
+bool RenderDoc::DoesTrackedFileExist(const rdcstr &nickname) const
+{
+  SCOPED_READLOCK(m_TrackedFilesLock);
+  for(const TrackedFile &f : m_TrackedFiles)
+  {
+    if(f.nickname == nickname)
+      return true;
+  }
+  return false;
+}
+
+// return false if the nickname already exists
+bool RenderDoc::AddTrackedFileReference(const rdcstr &nickname, const rdcstr &filepath)
+{
+  if(DoesTrackedFileExist(nickname))
+    return false;
+  SCOPED_WRITELOCK(m_TrackedFilesLock);
+  m_TrackedFiles.emplace_back(nickname, filepath);
+  return true;
+}
+
+void RenderDoc::ClearTrackedFiles()
+{
+  SCOPED_WRITELOCK(m_TrackedFilesLock);
+  m_TrackedFiles.clear();
+}
+
+bool RenderDoc::HasTrackedFileData() const
+{
+  SCOPED_READLOCK(m_TrackedFilesLock);
+  return !m_TrackedFiles.empty();
+}
+
+rdcarray<rdcstr> RenderDoc::GetTrackedFileNicknames() const
+{
+  rdcarray<rdcstr> nickNames;
+  {
+    SCOPED_READLOCK(m_TrackedFilesLock);
+    for(const TrackedFile &f : m_TrackedFiles)
+      nickNames.push_back(f.nickname);
+  }
+  return nickNames;
+}
+
+RDResult RenderDoc::ReadExternalFiles(RDCFile *rdc)
+{
+  int32_t idx = rdc->SectionIndex(SectionType::EmbeddedExternalFiles);
+  if(idx < 0)
+    RETURN_WARNING_RESULT(ResultCode::DataNotAvailable, "No EmbeddedExternalFiles section");
+
+  // int32_t countFileEntries;
+  int32_t countFileEntries = 0;
+  bytebuf sectionData;
+  {
+    StreamReader *reader = rdc->ReadSection(idx);
+    sectionData.resize((size_t)reader->GetSize());
+    bool success = reader->Read(sectionData.data(), reader->GetSize());
+    delete reader;
+    if(!success)
+      sectionData.clear();
+  }
+  const uint32_t bufferSize = (uint32_t)sectionData.size();
+  uint32_t readSize = sizeof(countFileEntries);
+  uint32_t byteIndex = 0;
+  if(byteIndex + readSize > bufferSize)
+    RETURN_WARNING_RESULT(ResultCode::FileCorrupted,
+                          "EmbeddedExternalFiles section does not have valid data");
+
+  const byte *bufferPtr = sectionData.data();
+  memcpy(&countFileEntries, bufferPtr, readSize);
+  byteIndex += readSize;
+  bufferPtr += readSize;
+
+  rdcarray<TrackedFile> externalFiles;
+  externalFiles.resize(countFileEntries);
+  // FileEntry fileEntries[];
+  for(int32_t i = 0; i < countFileEntries; ++i)
+  {
+    TrackedFile &externalFile = externalFiles[i];
+    externalFile.filepath.clear();
+
+    // FileEntry:
+    // uint32_t nameSize;
+    uint32_t nameSize = 0;
+    readSize = sizeof(nameSize);
+    if(byteIndex + readSize > bufferSize)
+      RETURN_WARNING_RESULT(ResultCode::FileCorrupted,
+                            "EmbeddedExternalFiles section does not have valid data");
+    memcpy(&nameSize, bufferPtr, readSize);
+    byteIndex += readSize;
+    bufferPtr += readSize;
+
+    // char name[];
+    readSize = nameSize;
+    if(byteIndex + readSize > bufferSize)
+      RETURN_WARNING_RESULT(ResultCode::FileCorrupted,
+                            "EmbeddedExternalFiles section does not have valid data");
+    externalFile.nickname.assign((const char *)bufferPtr, readSize);
+    byteIndex += readSize;
+    bufferPtr += readSize;
+
+    // uint32_t dataSize;
+    uint32_t dataSize = 0;
+    readSize = sizeof(dataSize);
+    if(byteIndex + readSize > bufferSize)
+      RETURN_WARNING_RESULT(ResultCode::FileCorrupted,
+                            "EmbeddedExternalFiles section does not have valid data");
+    memcpy(&dataSize, bufferPtr, readSize);
+    byteIndex += readSize;
+    bufferPtr += readSize;
+
+    // byte data[];
+    readSize = dataSize;
+    if(byteIndex + readSize > bufferSize)
+      RETURN_WARNING_RESULT(ResultCode::FileCorrupted,
+                            "EmbeddedExternalFiles section does not have valid data");
+    externalFile.data.assign(bufferPtr, readSize);
+    byteIndex += readSize;
+    bufferPtr += readSize;
+  }
+  if(byteIndex != bufferSize)
+    RETURN_WARNING_RESULT(ResultCode::FileCorrupted,
+                          "EmbeddedExternalFiles section does not have valid data");
+
+  SCOPED_WRITELOCK(m_TrackedFilesLock);
+  for(const TrackedFile &f : externalFiles)
+  {
+    for(TrackedFile &file : m_TrackedFiles)
+    {
+      if(file.nickname == f.nickname)
+      {
+        file.data = f.data;
+        file.filepath.clear();
+        break;
+      }
+    }
+    m_TrackedFiles.emplace_back(f.nickname, f.data);
+  }
+
+  return RDResult();
+}
+
+RDResult RenderDoc::WriteExternalFiles(RDCFile *rdc, const rdcarray<TrackedFile> &trackedFiles)
+{
+  if(!rdc)
+    RETURN_WARNING_RESULT(ResultCode::FileCorrupted,
+                          "Data missing for creation of file, set metadata first.");
+
+  RDResult rdcRes = rdc->Error();
+  if(rdcRes != ResultCode::Succeeded)
+    return rdcRes;
+
+  int32_t countFileEntries = trackedFiles.count();
+
+  bytebuf sectionData;
+
+  // int32_t countFileEntries;
+  sectionData.append((byte *)&countFileEntries, sizeof(countFileEntries));
+  // FileEntry fileEntries[];
+  for(const RenderDoc::TrackedFile &f : trackedFiles)
+  {
+    // FileEntry:
+    // uint32_t nameSize;
+    // char name[];
+    const rdcstr &name = f.nickname;
+    const uint32_t nameSize = (uint32_t)name.size();
+    sectionData.append((byte *)&nameSize, sizeof(nameSize));
+    sectionData.append((byte *)name.data(), nameSize);
+
+    // uint32_t dataSize;
+    // byte data[];
+    uint32_t dataSize = 0;
+    const byte *data = NULL;
+    bytebuf fileData;
+    if(!f.filepath.empty())
+    {
+      if(FileIO::ReadAll(f.filepath, fileData))
+      {
+        dataSize = (uint32_t)fileData.size();
+        data = fileData.data();
+      }
+      else
+      {
+        RDCWARN("Data missing for externally referenced file %s", f.filepath.c_str());
+      }
+    }
+    else
+    {
+      dataSize = (uint32_t)f.data.size();
+      data = f.data.data();
+    }
+    sectionData.append((byte *)&dataSize, sizeof(dataSize));
+    sectionData.append(data, dataSize);
+  }
+
+  SectionProperties props;
+  props.type = SectionType::EmbeddedExternalFiles;
+  props.flags = SectionFlags::ZstdCompressed;
+  props.version = 1;
+
+  StreamWriter *writer = rdc->WriteSection(props);
+  rdcRes = rdc->Error();
+  if(!writer || rdcRes != ResultCode::Succeeded)
+    return rdcRes;
+
+  writer->Write(sectionData.data(), sectionData.size());
+  writer->Finish();
+
+  delete writer;
+
+  return RDResult();
+}
+
+RDResult RenderDoc::EmbedExternalFiles(RDCFile *rdc)
+{
+  if(!rdc)
+    RETURN_WARNING_RESULT(ResultCode::FileCorrupted,
+                          "Data missing for creation of file, set metadata first.");
+
+  RDResult rdcRes = rdc->Error();
+  if(rdcRes != ResultCode::Succeeded)
+    return rdcRes;
+
+  if(rdc->SectionIndex(SectionType::EmbeddedExternalFiles) >= 0)
+    RDCWARN("Capture already has embedded external files - replacing existing section.");
+
+  SCOPED_WRITELOCK(m_TrackedFilesLock);
+  const rdcarray<RenderDoc::TrackedFile> &trackedFiles = m_TrackedFiles;
+  if(trackedFiles.empty())
+    RDCWARN("No external files to embed.");
+
+  RDCLOG("Embedding %d external files", trackedFiles.count());
+  return RenderDoc::Inst().WriteExternalFiles(rdc, trackedFiles);
+}
+
+RDResult RenderDoc::RemoveExternalFiles(RDCFile *rdc)
+{
+  if(!rdc)
+    RETURN_WARNING_RESULT(ResultCode::FileCorrupted,
+                          "Data missing for creation of file, set metadata first.");
+
+  RDResult rdcRes = rdc->Error();
+  if(rdcRes != ResultCode::Succeeded)
+    return rdcRes;
+
+  RDResult result;
+  if(rdc->SectionIndex(SectionType::EmbeddedExternalFiles) < 0)
+    RETURN_WARNING_RESULT(ResultCode::DataNotAvailable,
+                          "Capture does not have any embedded external files.");
+
+  RDCLOG("Removing embedded external files (setting count to 0)");
+  rdcarray<RenderDoc::TrackedFile> trackedFiles;
+  return RenderDoc::Inst().WriteExternalFiles(rdc, trackedFiles);
+}
+
+bool RenderDoc::HasEmbeddedFiles(RDCFile *rdc) const
+{
+  if(!rdc)
+    return false;
+
+  RDResult rdcRes = rdc->Error();
+  if(rdcRes != ResultCode::Succeeded)
+    return false;
+
+  int32_t idx = rdc->SectionIndex(SectionType::EmbeddedExternalFiles);
+  if(idx < 0)
+    return false;
+
+  int32_t countFileEntries = 0;
+  bytebuf sectionData;
+  {
+    StreamReader *reader = rdc->ReadSection(idx);
+    sectionData.resize((size_t)reader->GetSize());
+    bool success = reader->Read(sectionData.data(), reader->GetSize());
+    delete reader;
+    if(!success)
+      sectionData.clear();
+  }
+  if(sectionData.size() < sizeof(countFileEntries))
+    return false;
+
+  memcpy(&countFileEntries, sectionData.data(), sizeof(countFileEntries));
+  return countFileEntries > 0;
+}
+
 #if ENABLED(ENABLE_UNIT_TESTS)
 
 #undef None

renderdoc/core/core.h

@@ -614,11 +614,34 @@ public:
   void CycleActiveWindow();
   uint32_t GetCapturableWindowCount();
 
+  bool GetTrackedFileData(const rdcstr &nickname, bytebuf &data) const;
+  bool AddTrackedFileReference(const rdcstr &nickname, const rdcstr &filepath);
+  void ClearTrackedFiles();
+  bool HasTrackedFileData() const;
+  rdcarray<rdcstr> GetTrackedFileNicknames() const;
+
+  RDResult EmbedExternalFiles(RDCFile *rdc);
+  RDResult RemoveExternalFiles(RDCFile *rdc);
+  bool HasEmbeddedFiles(RDCFile *rdc) const;
+  RDResult ReadExternalFiles(RDCFile *rdc);
+
 private:
   RenderDoc();
   ~RenderDoc();
 
+  struct TrackedFile
+  {
+    TrackedFile() = default;
+    TrackedFile(const rdcstr &name, const rdcstr &path) : nickname(name), filepath(path) {}
+    TrackedFile(const rdcstr &name, const bytebuf &contents) : nickname(name), data(contents) {}
+    rdcstr nickname;
+    rdcstr filepath;
+    bytebuf data;
+  };
+
   void SyncAvailableGPUThread();
+  RDResult WriteExternalFiles(RDCFile *rdc, const rdcarray<TrackedFile> &trackedFiles);
+  bool DoesTrackedFileExist(const rdcstr &nickname) const;
 
   bool m_Replay;
 
@@ -720,6 +743,9 @@ private:
   ICrashHandler *m_ExHandler;
   Threading::RWLock m_ExHandlerLock;
 
+  mutable Threading::RWLock m_TrackedFilesLock;
+  rdcarray<TrackedFile> m_TrackedFiles;
+
   void ProcessConfig();
 
   SDObject *FindConfigSetting(const rdcstr &name);