Implement shader edit & replace for D3D12

renderdoc/driver/d3d12/d3d12_debug.cpp

@@ -1775,6 +1775,53 @@ void D3D12DebugManager::FillCBufferVariables(const vector<DXBC::CBufferVariable>
     outvars.push_back(v[i]);
 }
 
+void D3D12DebugManager::BuildShader(string source, string entry, const uint32_t compileFlags,
+                                    ShaderStageType type, ResourceId *id, string *errors)
+{
+  if(id == NULL || errors == NULL)
+  {
+    if(id)
+      *id = ResourceId();
+    return;
+  }
+
+  char *profile = NULL;
+
+  switch(type)
+  {
+    case eShaderStage_Vertex: profile = "vs_5_0"; break;
+    case eShaderStage_Hull: profile = "hs_5_0"; break;
+    case eShaderStage_Domain: profile = "ds_5_0"; break;
+    case eShaderStage_Geometry: profile = "gs_5_0"; break;
+    case eShaderStage_Pixel: profile = "ps_5_0"; break;
+    case eShaderStage_Compute: profile = "cs_5_0"; break;
+    default:
+      RDCERR("Unexpected type in BuildShader!");
+      *id = ResourceId();
+      return;
+  }
+
+  ID3DBlob *blob = NULL;
+  *errors = GetShaderBlob(source.c_str(), entry.c_str(), compileFlags, profile, &blob);
+
+  if(blob == NULL)
+  {
+    *id = ResourceId();
+    return;
+  }
+
+  D3D12_SHADER_BYTECODE byteCode;
+  byteCode.BytecodeLength = blob->GetBufferSize();
+  byteCode.pShaderBytecode = blob->GetBufferPointer();
+
+  WrappedID3D12PipelineState::ShaderEntry *sh =
+      WrappedID3D12PipelineState::AddShader(byteCode, m_WrappedDevice, NULL);
+
+  SAFE_RELEASE(blob);
+
+  *id = sh->GetResourceID();
+}
+
 void D3D12DebugManager::GetBufferData(ResourceId buff, uint64_t offset, uint64_t length,
                                       vector<byte> &retData)
 {

renderdoc/driver/d3d12/d3d12_debug.h

@@ -83,6 +83,9 @@ public:
   void GetBufferData(ResourceId buff, uint64_t offset, uint64_t length, vector<byte> &retData);
   void GetBufferData(ID3D12Resource *buff, uint64_t offset, uint64_t length, vector<byte> &retData);
 
+  void BuildShader(string source, string entry, const uint32_t compileFlags, ShaderStageType type,
+                   ResourceId *id, string *errors);
+
   D3D12_CPU_DESCRIPTOR_HANDLE AllocRTV();
   void FreeRTV(D3D12_CPU_DESCRIPTOR_HANDLE handle);
 

renderdoc/driver/d3d12/d3d12_device.cpp

@@ -1797,6 +1797,9 @@ void WrappedID3D12Device::DestroyInternalResources()
   for(size_t i = 0; i < m_InternalCmds.pendingcmds.size(); i++)
     SAFE_RELEASE(m_InternalCmds.pendingcmds[i]);
 
+  delete m_DebugManager;
+  m_DebugManager = NULL;
+
   SAFE_RELEASE(m_Alloc);
   SAFE_RELEASE(m_GPUSyncFence);
   CloseHandle(m_GPUSyncHandle);

renderdoc/driver/d3d12/d3d12_device_wrap.cpp

@@ -253,6 +253,9 @@ bool WrappedID3D12Device::Serialise_CreateGraphicsPipelineState(
 
       wrapped->graphics = new D3D12_GRAPHICS_PIPELINE_STATE_DESC(Descriptor);
 
+      wrapped->graphics->pRootSignature =
+          (ID3D12RootSignature *)GetResourceManager()->GetWrapper(wrapped->graphics->pRootSignature);
+
       D3D12_SHADER_BYTECODE *shaders[] = {
           &wrapped->graphics->VS, &wrapped->graphics->HS, &wrapped->graphics->DS,
           &wrapped->graphics->GS, &wrapped->graphics->PS,
@@ -263,7 +266,8 @@ bool WrappedID3D12Device::Serialise_CreateGraphicsPipelineState(
         if(shaders[i]->BytecodeLength == 0)
           shaders[i]->pShaderBytecode = NULL;
         else
-          shaders[i]->pShaderBytecode = WrappedID3D12PipelineState::AddShader(*shaders[i], this);
+          shaders[i]->pShaderBytecode =
+              WrappedID3D12PipelineState::AddShader(*shaders[i], this, wrapped);
       }
 
       GetResourceManager()->AddLiveResource(Pipe, ret);
@@ -311,6 +315,22 @@ HRESULT WrappedID3D12Device::CreateGraphicsPipelineState(const D3D12_GRAPHICS_PI
     else
     {
       GetResourceManager()->AddLiveResource(wrapped->GetResourceID(), wrapped);
+
+      wrapped->graphics = new D3D12_GRAPHICS_PIPELINE_STATE_DESC(*pDesc);
+
+      D3D12_SHADER_BYTECODE *shaders[] = {
+          &wrapped->graphics->VS, &wrapped->graphics->HS, &wrapped->graphics->DS,
+          &wrapped->graphics->GS, &wrapped->graphics->PS,
+      };
+
+      for(size_t i = 0; i < ARRAY_COUNT(shaders); i++)
+      {
+        if(shaders[i]->BytecodeLength == 0)
+          shaders[i]->pShaderBytecode = NULL;
+        else
+          shaders[i]->pShaderBytecode =
+              WrappedID3D12PipelineState::AddShader(*shaders[i], this, NULL);
+      }
     }
 
     *ppPipelineState = (ID3D12PipelineState *)wrapped;
@@ -344,8 +364,11 @@ bool WrappedID3D12Device::Serialise_CreateComputePipelineState(
 
       wrapped->compute = new D3D12_COMPUTE_PIPELINE_STATE_DESC(Descriptor);
 
+      wrapped->compute->pRootSignature =
+          (ID3D12RootSignature *)GetResourceManager()->GetWrapper(wrapped->compute->pRootSignature);
+
       wrapped->compute->CS.pShaderBytecode =
-          WrappedID3D12PipelineState::AddShader(wrapped->compute->CS, this);
+          WrappedID3D12PipelineState::AddShader(wrapped->compute->CS, this, wrapped);
 
       GetResourceManager()->AddLiveResource(Pipe, ret);
     }
@@ -392,6 +415,11 @@ HRESULT WrappedID3D12Device::CreateComputePipelineState(const D3D12_COMPUTE_PIPE
     else
     {
       GetResourceManager()->AddLiveResource(wrapped->GetResourceID(), wrapped);
+
+      wrapped->compute = new D3D12_COMPUTE_PIPELINE_STATE_DESC(*pDesc);
+
+      wrapped->compute->CS.pShaderBytecode =
+          WrappedID3D12PipelineState::AddShader(wrapped->compute->CS, this, NULL);
     }
 
     *ppPipelineState = (ID3D12PipelineState *)wrapped;

renderdoc/driver/d3d12/d3d12_replay.cpp

@@ -23,6 +23,7 @@
  ******************************************************************************/
 
 #include "d3d12_replay.h"
+#include "driver/dx/official/d3dcompiler.h"
 #include "driver/dxgi/dxgi_common.h"
 #include "d3d12_command_queue.h"
 #include "d3d12_device.h"
@@ -1311,16 +1312,6 @@ void D3D12Replay::FlipOutputWindow(uint64_t id)
   m_pDevice->GetDebugManager()->FlipOutputWindow(id);
 }
 
-void D3D12Replay::ReplaceResource(ResourceId from, ResourceId to)
-{
-  m_pDevice->GetResourceManager()->ReplaceResource(from, to);
-}
-
-void D3D12Replay::RemoveReplacement(ResourceId id)
-{
-  m_pDevice->GetResourceManager()->RemoveReplacement(id);
-}
-
 void D3D12Replay::InitCallstackResolver()
 {
   m_pDevice->GetMainSerialiser()->InitCallstackResolver();
@@ -1341,6 +1332,111 @@ vector<DebugMessage> D3D12Replay::GetDebugMessages()
   return m_pDevice->GetDebugMessages();
 }
 
+void D3D12Replay::BuildTargetShader(string source, string entry, const uint32_t compileFlags,
+                                    ShaderStageType type, ResourceId *id, string *errors)
+{
+  m_pDevice->GetDebugManager()->BuildShader(source, entry, D3DCOMPILE_DEBUG | compileFlags, type,
+                                            id, errors);
+}
+
+void D3D12Replay::ReplaceResource(ResourceId from, ResourceId to)
+{
+  D3D12ResourceManager *rm = m_pDevice->GetResourceManager();
+
+  // remove any previous replacement
+  RemoveReplacement(from);
+
+  if(rm->HasLiveResource(from))
+  {
+    ID3D12DeviceChild *resource = rm->GetLiveResource(from);
+
+    if(WrappedID3D12PipelineState::ShaderEntry::IsAlloc(resource))
+    {
+      WrappedID3D12PipelineState::ShaderEntry *sh =
+          (WrappedID3D12PipelineState::ShaderEntry *)resource;
+
+      for(size_t i = 0; i < sh->m_Pipes.size(); i++)
+      {
+        WrappedID3D12PipelineState *pipe = sh->m_Pipes[i];
+
+        ResourceId id = rm->GetOriginalID(pipe->GetResourceID());
+
+        ID3D12PipelineState *replpipe = NULL;
+
+        D3D12_SHADER_BYTECODE shDesc =
+            rm->GetLiveAs<WrappedID3D12PipelineState::ShaderEntry>(to)->GetDesc();
+
+        if(pipe->graphics)
+        {
+          D3D12_GRAPHICS_PIPELINE_STATE_DESC desc = *pipe->graphics;
+
+          D3D12_SHADER_BYTECODE *shaders[] = {
+              &desc.VS, &desc.HS, &desc.DS, &desc.GS, &desc.PS,
+          };
+
+          for(size_t i = 0; i < ARRAY_COUNT(shaders); i++)
+          {
+            if(shaders[i]->BytecodeLength > 0)
+            {
+              WrappedID3D12PipelineState::ShaderEntry *s =
+                  (WrappedID3D12PipelineState::ShaderEntry *)shaders[i]->pShaderBytecode;
+
+              if(s->GetResourceID() == from)
+                *shaders[i] = shDesc;
+              else
+                *shaders[i] = s->GetDesc();
+            }
+          }
+
+          m_pDevice->CreateGraphicsPipelineState(&desc, __uuidof(ID3D12PipelineState),
+                                                 (void **)&replpipe);
+        }
+        else
+        {
+          D3D12_COMPUTE_PIPELINE_STATE_DESC desc = *pipe->compute;
+
+          // replace the shader
+          desc.CS = shDesc;
+
+          m_pDevice->CreateComputePipelineState(&desc, __uuidof(ID3D12PipelineState),
+                                                (void **)&replpipe);
+        }
+
+        rm->ReplaceResource(id, GetResID(replpipe));
+      }
+    }
+
+    rm->ReplaceResource(from, to);
+  }
+}
+
+void D3D12Replay::RemoveReplacement(ResourceId id)
+{
+  D3D12ResourceManager *rm = m_pDevice->GetResourceManager();
+
+  rm->RemoveReplacement(id);
+
+  if(rm->HasLiveResource(id))
+  {
+    ID3D12DeviceChild *resource = rm->GetLiveResource(id);
+
+    if(WrappedID3D12PipelineState::ShaderEntry::IsAlloc(resource))
+    {
+      WrappedID3D12PipelineState::ShaderEntry *sh =
+          (WrappedID3D12PipelineState::ShaderEntry *)resource;
+
+      for(size_t i = 0; i < sh->m_Pipes.size(); i++)
+      {
+        WrappedID3D12PipelineState *pipe = sh->m_Pipes[i];
+
+        ResourceId id = rm->GetOriginalID(pipe->GetResourceID());
+
+        rm->RemoveReplacement(id);
+      }
+    }
+  }
+}
+
 #pragma region not yet implemented
 
 vector<uint32_t> D3D12Replay::GetPassEvents(uint32_t eventID)
@@ -1406,11 +1502,6 @@ void D3D12Replay::RenderMesh(uint32_t eventID, const vector<MeshFormat> &seconda
 {
 }
 
-void D3D12Replay::BuildTargetShader(string source, string entry, const uint32_t compileFlags,
-                                    ShaderStageType type, ResourceId *id, string *errors)
-{
-}
-
 void D3D12Replay::BuildCustomShader(string source, string entry, const uint32_t compileFlags,
                                     ShaderStageType type, ResourceId *id, string *errors)
 {

renderdoc/driver/d3d12/d3d12_resources.h

@@ -497,6 +497,14 @@ public:
       m_DebugInfoPath = path;
     }
 
+    D3D12_SHADER_BYTECODE GetDesc()
+    {
+      D3D12_SHADER_BYTECODE ret;
+      ret.BytecodeLength = m_Bytecode.size();
+      ret.pShaderBytecode = (const void *)&m_Bytecode[0];
+      return ret;
+    }
+
     DXBC::DXBCFile *GetDXBC()
     {
       if(m_DXBCFile == NULL && !m_Bytecode.empty())
@@ -521,6 +529,8 @@ public:
       return m_Mapping;
     }
 
+    vector<WrappedID3D12PipelineState *> m_Pipes;
+
   private:
     ShaderEntry(const ShaderEntry &e);
     void TryReplaceOriginalByteCode();
@@ -544,7 +554,8 @@ public:
     TypeEnum = Resource_PipelineState,
   };
 
-  static ShaderEntry *AddShader(const D3D12_SHADER_BYTECODE &byteCode, WrappedID3D12Device *device)
+  static ShaderEntry *AddShader(const D3D12_SHADER_BYTECODE &byteCode, WrappedID3D12Device *device,
+                                WrappedID3D12PipelineState *pipeline)
   {
     DXBCKey key(byteCode);
     ShaderEntry *shader = m_Shaders[key];
@@ -554,6 +565,10 @@ public:
     else
       shader->AddRef();
 
+    if(pipeline &&
+       std::find(shader->m_Pipes.begin(), shader->m_Pipes.end(), pipeline) == shader->m_Pipes.end())
+      shader->m_Pipes.push_back(pipeline);
+
     return shader;
   }
 

renderdocui/Windows/PipelineState/D3D11PipelineStateViewer.cs

@@ -2367,7 +2367,7 @@ namespace renderdocui.Windows.PipelineState
 
             if (stage.Shader == ResourceId.Null || shaderDetails == null) return;
 
-            var entryFunc = String.Format("EditedShader{0}S", stage.stage.ToString()[0]);
+            var entryFunc = String.Format("EditedShader{0}S", stage.stage.Str(GraphicsAPI.D3D11)[0]);
 
             string mainfile = "";
 

renderdocui/Windows/PipelineState/D3D12PipelineStateViewer.cs

@@ -2023,7 +2023,7 @@ namespace renderdocui.Windows.PipelineState
 
             if (stage.Shader == ResourceId.Null || shaderDetails == null) return;
 
-            var entryFunc = String.Format("EditedShader{0}S", stage.stage.ToString()[0]);
+            var entryFunc = String.Format("EditedShader{0}S", stage.stage.Str(GraphicsAPI.D3D12)[0]);
 
             string mainfile = "";