ICaptureAccess APIs for externally referenced dependency files

ResultDetails EmbedDependentFilesIntoCapture();
ResultDetails RemoveDependentFilesFromCapture();
int32_t CountEmbeddedDependentFiles();
int32_t CountExternallyReferencedFiles();
rdcarray<rdcstr> GetExternallyReferencedFiles();

renderdoc/api/replay/renderdoc_replay.h

@@ -1343,6 +1343,63 @@ Must only be called after :meth:`InitResolver` has returned ``True``.
 )");
   virtual rdcstr DriverName() = 0;
 
+  DOCUMENT(R"(Stores the dependent file data into the capture i.e. shader debug files.
+
+This reads the contents of the dependent files and stores their file contents into the capture.
+This can help the capture to be more portable by embedding all externally referenced dependent files.
+Use :meth:`RemoveDependenciesFromCapture` to remove the embedded file data.
+
+.. warning::
+  Will remove all the existing embedded file data from the capture.
+  Will directly modify the capture file on disk.
+
+.. note::
+  This will increase the size of the capture file.
+  Externally referenced files which can't be found on disk are skipped.
+
+:return: The result of the operation.
+:rtype: ResultDetails
+)");
+  virtual ResultDetails EmbedDependenciesIntoCapture() = 0;
+
+  DOCUMENT(R"(Removes the dependent files storage from the capture i.e. shader debug files.
+
+The files will be still be considered to be referenced by the capture and could be re-embedded 
+by calling :meth:`EmbedDependenciesIntoCapture`.
+
+.. warning::
+  Will directly modify the capture file on disk.
+
+:return: The result of the operation.
+:rtype: ResultDetails
+)");
+  virtual ResultDetails RemoveDependenciesFromCapture() = 0;
+
+  DOCUMENT(R"(Are there any depdendent files embedded in the capture i.e. shader debug files.
+
+:return: ``True`` if the capture has embedded dependent files, or ``False`` if the capture does not any embedded dependent files.
+:rtype: bool
+)");
+  virtual bool HasEmbeddedDependencies() = 0;
+
+  DOCUMENT(R"(Does the capture have references to dependecies i.e. shader debug files.
+
+:return: ``True`` if the capture has references to dependent files, or ``False`` if the capture does not contain references to dependent files.
+:rtype: bool
+)");
+  virtual bool HasPendingDependencies() = 0;
+
+  DOCUMENT(R"(Retrieve a list of the nicknames of the externally referenced dependent files being referenced 
+by the capture i.e. shader debug files.
+
+.. note::
+  The nicknames can be arbitary and do not have to be a filename or a file path.
+
+:return: A list of the nicknames used to reference dependencies.
+:rtype: List[str]
+)");
+  virtual rdcarray<rdcstr> GetPendingDependenciesNicknames() = 0;
+
 protected:
   ICaptureAccess() = default;
   ~ICaptureAccess() = default;

renderdoc/core/remote_server.cpp

@@ -87,6 +87,11 @@ enum RemoteServerPacket
   eRemoteServer_WriteSection,
   eRemoteServer_GetAvailableGPUs,
   eRemoteServer_RemoteServerCount,
+  eRemoteServer_EmbedDependenciesIntoCapture,
+  eRemoteServer_RemoveDependenciesFromCapture,
+  eRemoteServer_HasEmbeddedDependencies,
+  eRemoteServer_HasPendingDependencies,
+  eRemoteServer_GetPendingDependenciesNicknames,
 };
 
 DECLARE_REFLECTION_ENUM(RemoteServerPacket);
@@ -131,6 +136,14 @@ rdcstr DoStringise(const RemoteServerPacket &el)
     STRINGISE_ENUM_NAMED(eRemoteServer_WriteSection, "WriteSection");
     STRINGISE_ENUM_NAMED(eRemoteServer_GetAvailableGPUs, "GetAvailableGPUs");
     STRINGISE_ENUM_NAMED(eRemoteServer_RemoteServerCount, "RemoteServerCount");
+    STRINGISE_ENUM_NAMED(eRemoteServer_EmbedDependenciesIntoCapture,
+                         "EmbedDependenciesIntoCapture");
+    STRINGISE_ENUM_NAMED(eRemoteServer_RemoveDependenciesFromCapture,
+                         "RemoveDependenciesFromCapture");
+    STRINGISE_ENUM_NAMED(eRemoteServer_HasEmbeddedDependencies, "HasEmbeddedDependencies");
+    STRINGISE_ENUM_NAMED(eRemoteServer_HasPendingDependencies, "HasPendingDependencies");
+    STRINGISE_ENUM_NAMED(eRemoteServer_GetPendingDependenciesNicknames,
+                         "GetPendingDependenciesNicknames");
   }
   END_ENUM_STRINGISE();
 }
@@ -537,6 +550,18 @@ static void ActiveRemoteClientThread(ClientThread *threadData,
             }
           });
 
+          // This has to be before the driver is created for the capture
+          RenderDoc::Inst().ClearTrackedFiles();
+          if(rdc->SectionIndex(SectionType::EmbeddedExternalFiles) >= 0)
+          {
+            ResultDetails ret = RenderDoc::Inst().ReadExternalFiles(rdc);
+            if(!ret.OK())
+            {
+              RDCERR("ReadExternalFiles failed Code:'%s' Message:'%s'", ToStr(ret.code).c_str(),
+                     ret.Message().c_str());
+            }
+          }
+
           // if we have a replay driver, try to create it so we can display a local preview e.g.
           if(RenderDoc::Inst().HasReplayDriver(rdc->GetDriver()))
           {
@@ -897,6 +922,108 @@ static void ActiveRemoteClientThread(ClientThread *threadData,
         SERIALISE_ELEMENT(ident);
       }
     }
+    else if(type == eRemoteServer_EmbedDependenciesIntoCapture)
+    {
+      reader.EndChunk();
+
+      RDResult result;
+      if(rdc)
+      {
+        result = RenderDoc::Inst().EmbedExternalFiles(rdc);
+      }
+      else
+      {
+        SET_ERROR_RESULT(result, ResultCode::InternalError,
+                         "Attempt to embed external dependency files with no capture open");
+      }
+
+      {
+        WRITE_DATA_SCOPE();
+        SCOPED_SERIALISE_CHUNK(eRemoteServer_EmbedDependenciesIntoCapture);
+        SERIALISE_ELEMENT(result);
+      }
+    }
+    else if(type == eRemoteServer_RemoveDependenciesFromCapture)
+    {
+      reader.EndChunk();
+
+      RDResult result;
+      if(rdc)
+      {
+        result = RenderDoc::Inst().RemoveExternalFiles(rdc);
+      }
+      else
+      {
+        SET_ERROR_RESULT(result, ResultCode::InternalError,
+                         "Attempt to remove external files with no capture open");
+      }
+
+      {
+        WRITE_DATA_SCOPE();
+        SCOPED_SERIALISE_CHUNK(eRemoteServer_RemoveDependenciesFromCapture);
+        SERIALISE_ELEMENT(result);
+      }
+    }
+    else if(type == eRemoteServer_HasEmbeddedDependencies)
+    {
+      reader.EndChunk();
+
+      bool res = false;
+      if(rdc)
+      {
+        res = RenderDoc::Inst().HasEmbeddedFiles(rdc);
+      }
+      else
+      {
+        RDCWARN("Attempt to check for embedded files with no capture open");
+      }
+
+      {
+        WRITE_DATA_SCOPE();
+        SCOPED_SERIALISE_CHUNK(eRemoteServer_HasEmbeddedDependencies);
+        SERIALISE_ELEMENT(res);
+      }
+    }
+    else if(type == eRemoteServer_HasPendingDependencies)
+    {
+      reader.EndChunk();
+
+      bool res = false;
+      if(rdc)
+      {
+        res = RenderDoc::Inst().HasTrackedFileData();
+      }
+      else
+      {
+        RDCWARN("Attempt to check for externally referenced files with no capture open");
+      }
+
+      {
+        WRITE_DATA_SCOPE();
+        SCOPED_SERIALISE_CHUNK(eRemoteServer_HasPendingDependencies);
+        SERIALISE_ELEMENT(res);
+      }
+    }
+    else if(type == eRemoteServer_GetPendingDependenciesNicknames)
+    {
+      reader.EndChunk();
+
+      rdcarray<rdcstr> res;
+      if(rdc)
+      {
+        res = RenderDoc::Inst().GetTrackedFileNicknames();
+      }
+      else
+      {
+        RDCWARN("Attempt to get nickanmes of externally referenced files with no capture open");
+      }
+
+      {
+        WRITE_DATA_SCOPE();
+        SCOPED_SERIALISE_CHUNK(eRemoteServer_GetPendingDependenciesNicknames);
+        SERIALISE_ELEMENT(res);
+      }
+    }
     else if((int)type >= eReplayProxy_First && proxy)
     {
       bool ok = proxy->Tick(type);
@@ -1990,7 +2117,7 @@ ResultDetails RemoteServer::WriteSection(const SectionProperties &props, const b
     }
     else
     {
-      RDCERR("Unexpected response to has write section request");
+      RDCERR("Unexpected response to write section request");
     }
 
     ser.EndChunk();
@@ -2109,3 +2236,162 @@ rdcarray<rdcstr> RemoteServer::GetResolve(const rdcarray<uint64_t> &callstack)
 
   return StackFrames;
 }
+
+ResultDetails RemoteServer::EmbedDependenciesIntoCapture()
+{
+  RDResult ret;
+
+  if(!Connected())
+  {
+    ret.code = ResultCode::RemoteServerConnectionLost;
+    return ret;
+  }
+
+  {
+    WRITE_DATA_SCOPE();
+    SCOPED_SERIALISE_CHUNK(eRemoteServer_EmbedDependenciesIntoCapture);
+  }
+
+  {
+    READ_DATA_SCOPE();
+    RemoteServerPacket type = ser.ReadChunk<RemoteServerPacket>();
+
+    if(type == eRemoteServer_EmbedDependenciesIntoCapture)
+    {
+      SERIALISE_ELEMENT(ret);
+    }
+    else
+    {
+      RDCERR("Unexpected response to embed dependencies into capture request");
+    }
+
+    ser.EndChunk();
+  }
+
+  return ret;
+}
+
+ResultDetails RemoteServer::RemoveDependenciesFromCapture()
+{
+  RDResult ret;
+  if(!Connected())
+  {
+    ret.code = ResultCode::RemoteServerConnectionLost;
+    return ret;
+  }
+
+  {
+    WRITE_DATA_SCOPE();
+    SCOPED_SERIALISE_CHUNK(eRemoteServer_RemoveDependenciesFromCapture);
+  }
+
+  {
+    READ_DATA_SCOPE();
+    RemoteServerPacket type = ser.ReadChunk<RemoteServerPacket>();
+
+    if(type == eRemoteServer_RemoveDependenciesFromCapture)
+    {
+      SERIALISE_ELEMENT(ret);
+    }
+    else
+    {
+      RDCERR("Unexpected response to remove dependencies from capture request");
+    }
+
+    ser.EndChunk();
+  }
+
+  return ret;
+}
+
+bool RemoteServer::HasEmbeddedDependencies()
+{
+  bool ret = false;
+  if(!Connected())
+  {
+    return false;
+  }
+
+  {
+    WRITE_DATA_SCOPE();
+    SCOPED_SERIALISE_CHUNK(eRemoteServer_HasEmbeddedDependencies);
+  }
+
+  {
+    READ_DATA_SCOPE();
+    RemoteServerPacket type = ser.ReadChunk<RemoteServerPacket>();
+
+    if(type == eRemoteServer_HasEmbeddedDependencies)
+    {
+      SERIALISE_ELEMENT(ret);
+    }
+    else
+    {
+      RDCERR("Unexpected response to has embedded dependencies request");
+    }
+
+    ser.EndChunk();
+  }
+  return ret;
+}
+
+bool RemoteServer::HasPendingDependencies()
+{
+  bool ret = false;
+  if(!Connected())
+  {
+    return false;
+  }
+
+  {
+    WRITE_DATA_SCOPE();
+    SCOPED_SERIALISE_CHUNK(eRemoteServer_HasPendingDependencies);
+  }
+
+  {
+    READ_DATA_SCOPE();
+    RemoteServerPacket type = ser.ReadChunk<RemoteServerPacket>();
+
+    if(type == eRemoteServer_HasPendingDependencies)
+    {
+      SERIALISE_ELEMENT(ret);
+    }
+    else
+    {
+      RDCERR("Unexpected response to has pending dependencies request");
+    }
+
+    ser.EndChunk();
+  }
+  return ret;
+}
+
+rdcarray<rdcstr> RemoteServer::GetPendingDependenciesNicknames()
+{
+  rdcarray<rdcstr> ret;
+  if(!Connected())
+  {
+    return ret;
+  }
+  {
+    WRITE_DATA_SCOPE();
+    SCOPED_SERIALISE_CHUNK(eRemoteServer_GetPendingDependenciesNicknames);
+  }
+
+  {
+    READ_DATA_SCOPE();
+    RemoteServerPacket type = ser.ReadChunk<RemoteServerPacket>();
+
+    if(type == eRemoteServer_GetPendingDependenciesNicknames)
+    {
+      SERIALISE_ELEMENT(ret);
+    }
+    else
+    {
+      RDCERR("Unexpected response to get nicknmes of externally referenced files");
+    }
+
+    ser.EndChunk();
+  }
+  return ret;
+}

renderdoc/core/remote_server.h

@@ -100,6 +100,12 @@ public:
 
   virtual rdcarray<rdcstr> GetResolve(const rdcarray<uint64_t> &callstack);
 
+  virtual ResultDetails EmbedDependenciesIntoCapture();
+  virtual ResultDetails RemoveDependenciesFromCapture();
+  virtual bool HasEmbeddedDependencies();
+  virtual bool HasPendingDependencies();
+  virtual rdcarray<rdcstr> GetPendingDependenciesNicknames();
+
 protected:
   Network::Socket *m_Socket;
   WriteSerialiser *writer;

renderdoc/replay/capture_file.cpp

@@ -168,6 +168,12 @@ public:
   ResultDetails InitResolver(bool interactive, RENDERDOC_ProgressCallback progress);
   rdcarray<rdcstr> GetResolve(const rdcarray<uint64_t> &callstack);
 
+  ResultDetails EmbedDependenciesIntoCapture();
+  ResultDetails RemoveDependenciesFromCapture();
+  bool HasEmbeddedDependencies();
+  bool HasPendingDependencies();
+  rdcarray<rdcstr> GetPendingDependenciesNicknames();
+
 private:
   ResultDetails Init();
 
@@ -368,6 +374,12 @@ rdcpair<ResultDetails, IReplayController *> CaptureFile::OpenCapture(const Repla
 
   RenderDoc::Inst().SetProgressCallback<LoadProgress>(progress);
 
+  // This has to be before the device is created for the capture
+  RenderDoc::Inst().ClearTrackedFiles();
+  // This section is optional any errors do not block opening the capture
+  if(m_RDC->SectionIndex(SectionType::EmbeddedExternalFiles) >= 0)
+    ret = RenderDoc::Inst().ReadExternalFiles(m_RDC);
+
   ret = render->CreateDevice(m_RDC, opts);
 
   RenderDoc::Inst().SetProgressCallback<LoadProgress>(RENDERDOC_ProgressCallback());
@@ -851,6 +863,31 @@ rdcarray<rdcstr> CaptureFile::GetResolve(const rdcarray<uint64_t> &callstack)
   return ret;
 }
 
+ResultDetails CaptureFile::EmbedDependenciesIntoCapture()
+{
+  return RenderDoc::Inst().EmbedExternalFiles(m_RDC);
+}
+
+ResultDetails CaptureFile::RemoveDependenciesFromCapture()
+{
+  return RenderDoc::Inst().RemoveExternalFiles(m_RDC);
+}
+
+bool CaptureFile::HasEmbeddedDependencies()
+{
+  return RenderDoc::Inst().HasEmbeddedFiles(m_RDC);
+}
+
+bool CaptureFile::HasPendingDependencies()
+{
+  return RenderDoc::Inst().HasTrackedFileData();
+}
+
+rdcarray<rdcstr> CaptureFile::GetPendingDependenciesNicknames()
+{
+  return RenderDoc::Inst().GetTrackedFileNicknames();
+}
+
 extern "C" RENDERDOC_API ICaptureFile *RENDERDOC_CC RENDERDOC_OpenCaptureFile()
 {
   return new CaptureFile();

renderdoc/replay/replay_controller.cpp

@@ -1972,6 +1972,7 @@ void ReplayController::Shutdown()
     m_pDevice->Shutdown();
   m_pDevice = NULL;
 
+  RenderDoc::Inst().ClearTrackedFiles();
   delete this;
 }