Add shader edit & replace for vulkan (without source for now)

* Currently at least glslang doesn't emit OpSource with source data
  embedded, so we don't pull it out and for the moment we just pre-fill
  the shader editor with the disassembly text (which is sort of but not
  really GLSL) as a better-than-nothing default.

renderdoc/core/resource_manager.h

@@ -1357,6 +1357,9 @@ WrappedResourceType ResourceManager<WrappedResourceType, RealResourceType,
 {
   SCOPED_LOCK(m_Lock);
 
+  if(m_Replacements.find(id) != m_Replacements.end())
+    return GetCurrentResource(m_Replacements[id]);
+
   RDCASSERT(m_CurrentResourceMap.find(id) != m_CurrentResourceMap.end(), id);
   return m_CurrentResourceMap[id];
 }

renderdoc/driver/vulkan/vk_debug.cpp

@@ -2319,6 +2319,102 @@ void VulkanDebugManager::RenderTextInternal(const TextPrintState &textstate, flo
   ObjDisp(textstate.cmd)->CmdDraw(Unwrap(textstate.cmd), 4, (uint32_t)strlen(text), 0, 0);
 }
 
+void VulkanDebugManager::ReplaceResource(ResourceId from, ResourceId to)
+{
+  VkDevice dev = m_pDriver->GetDev();
+
+  // we're passed in the original ID but we want the live ID for comparison
+  ResourceId liveid = GetResourceManager()->GetLiveID(from);
+
+  VkShaderModule srcShaderModule = GetResourceManager()->GetCurrentHandle<VkShaderModule>(liveid);
+  VkShaderModule dstShaderModule = GetResourceManager()->GetCurrentHandle<VkShaderModule>(to);
+
+  // remake and replace any pipelines that referenced this shader
+  for(auto it = m_pDriver->m_CreationInfo.m_Pipeline.begin();
+      it != m_pDriver->m_CreationInfo.m_Pipeline.end(); ++it)
+  {
+    bool refdShader = false;
+    for(size_t i = 0; i < ARRAY_COUNT(it->second.shaders); i++)
+    {
+      if(it->second.shaders[i].module == liveid)
+      {
+        refdShader = true;
+        break;
+      }
+    }
+
+    if(refdShader)
+    {
+      VkGraphicsPipelineCreateInfo pipeCreateInfo;
+      MakeGraphicsPipelineInfo(pipeCreateInfo, it->first);
+
+      // replace the relevant module
+      for(uint32_t i = 0; i < pipeCreateInfo.stageCount; i++)
+      {
+        VkPipelineShaderStageCreateInfo &sh =
+            (VkPipelineShaderStageCreateInfo &)pipeCreateInfo.pStages[i];
+
+        if(sh.module == srcShaderModule)
+          sh.module = dstShaderModule;
+      }
+
+      // create the new pipeline
+      VkPipeline pipe = VK_NULL_HANDLE;
+      VkResult vkr =
+          m_pDriver->vkCreateGraphicsPipelines(dev, VK_NULL_HANDLE, 1, &pipeCreateInfo, NULL, &pipe);
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+      // remove the replacements
+      GetResourceManager()->ReplaceResource(it->first, GetResID(pipe));
+      GetResourceManager()->ReplaceResource(GetResourceManager()->GetOriginalID(it->first),
+                                            GetResID(pipe));
+    }
+  }
+
+  // make the actual shader module replacements
+  GetResourceManager()->ReplaceResource(from, to);
+  GetResourceManager()->ReplaceResource(liveid, to);
+}
+
+void VulkanDebugManager::RemoveReplacement(ResourceId id)
+{
+  VkDevice dev = m_pDriver->GetDev();
+
+  // we're passed in the original ID but we want the live ID for comparison
+  ResourceId liveid = GetResourceManager()->GetLiveID(id);
+
+  // remove the actual shader module replacements
+  GetResourceManager()->RemoveReplacement(id);
+  GetResourceManager()->RemoveReplacement(liveid);
+
+  // remove any replacements on pipelines that referenced this shader
+  for(auto it = m_pDriver->m_CreationInfo.m_Pipeline.begin();
+      it != m_pDriver->m_CreationInfo.m_Pipeline.end(); ++it)
+  {
+    bool refdShader = false;
+    for(size_t i = 0; i < ARRAY_COUNT(it->second.shaders); i++)
+    {
+      if(it->second.shaders[i].module == liveid)
+      {
+        refdShader = true;
+        break;
+      }
+    }
+
+    if(refdShader)
+    {
+      VkPipeline pipe = GetResourceManager()->GetCurrentHandle<VkPipeline>(it->first);
+
+      // delete the replacement pipeline
+      m_pDriver->vkDestroyPipeline(dev, pipe, NULL);
+
+      // remove both live and original replacements, since we will have made these above
+      GetResourceManager()->RemoveReplacement(it->first);
+      GetResourceManager()->RemoveReplacement(GetResourceManager()->GetOriginalID(it->first));
+    }
+  }
+}
+
 void VulkanDebugManager::CreateCustomShaderTex(uint32_t width, uint32_t height)
 {
   VkDevice dev = m_Device;

renderdoc/driver/vulkan/vk_debug.h

@@ -128,6 +128,9 @@ public:
   void CreateCustomShaderTex(uint32_t width, uint32_t height);
   void CreateCustomShaderPipeline(ResourceId shader);
 
+  void ReplaceResource(ResourceId from, ResourceId to);
+  void RemoveReplacement(ResourceId id);
+
   struct GPUBuffer
   {
     enum CreateFlags

renderdoc/driver/vulkan/vk_replay.cpp

@@ -5036,29 +5036,66 @@ ResourceId VulkanReplay::ApplyCustomShader(ResourceId shader, ResourceId texid,
 void VulkanReplay::BuildTargetShader(string source, string entry, const uint32_t compileFlags,
                                      ShaderStageType type, ResourceId *id, string *errors)
 {
-  VULKANNOTIMP("BuildTargetShader");
-  if(errors)
-    *errors = "Shader edit & replace is not yet supported on Vulkan";
-  if(id)
+  SPIRVShaderStage stage = eSPIRVInvalid;
+
+  switch(type)
+  {
+    case eShaderStage_Vertex: stage = eSPIRVVertex; break;
+    case eShaderStage_Hull: stage = eSPIRVTessControl; break;
+    case eShaderStage_Domain: stage = eSPIRVTessEvaluation; break;
+    case eShaderStage_Geometry: stage = eSPIRVGeometry; break;
+    case eShaderStage_Pixel: stage = eSPIRVFragment; break;
+    case eShaderStage_Compute: stage = eSPIRVCompute; break;
+    default:
+      RDCERR("Unexpected type in BuildShader!");
+      *id = ResourceId();
+      return;
+  }
+
+  vector<string> sources;
+  sources.push_back(source);
+  vector<uint32_t> spirv;
+
+  string output = CompileSPIRV(stage, sources, spirv);
+
+  if(spirv.empty())
+  {
     *id = ResourceId();
+    *errors = output;
+    return;
+  }
+
+  VkShaderModuleCreateInfo modinfo = {
+      VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,
+      NULL,
+      0,
+      spirv.size() * sizeof(uint32_t),
+      &spirv[0],
+  };
+
+  VkShaderModule module;
+  VkResult vkr = m_pDriver->vkCreateShaderModule(m_pDriver->GetDev(), &modinfo, NULL, &module);
+  RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+  *id = GetResID(module);
 }
 
 void VulkanReplay::ReplaceResource(ResourceId from, ResourceId to)
 {
-  // won't get hit until BuildTargetShader is implemented
-  VULKANNOTIMP("ReplaceResource");
+  GetDebugManager()->ReplaceResource(from, to);
 }
 
 void VulkanReplay::RemoveReplacement(ResourceId id)
 {
-  // won't get hit until BuildTargetShader is implemented
-  VULKANNOTIMP("RemoveReplacement");
+  GetDebugManager()->RemoveReplacement(id);
 }
 
 void VulkanReplay::FreeTargetResource(ResourceId id)
 {
-  // won't get hit until BuildTargetShader is implemented
-  VULKANNOTIMP("FreeTargetResource");
+  if(id == ResourceId())
+    return;
+
+  m_pDriver->ReleaseResource(GetResourceManager()->GetCurrentResource(id));
 }
 
 vector<PixelModification> VulkanReplay::PixelHistory(vector<EventUsage> events, ResourceId target,

renderdocui/Windows/PipelineState/VulkanPipelineStateViewer.cs

@@ -1925,10 +1925,71 @@ namespace renderdocui.Windows.PipelineState
             }
         }
 
-        // start a shaderviewer to edit this shader, optionally generating stub HLSL if there isn't
-        // HLSL source available for this shader.
+        // start a shaderviewer to edit this shader, optionally generating stub GLSL if there isn't
+        // GLSL source available for this shader.
         private void shaderedit_Click(object sender, EventArgs e)
         {
+            VulkanPipelineState.ShaderStage stage = GetStageForSender(sender);
+
+            if (stage == null) return;
+
+            ShaderReflection shaderDetails = stage.ShaderDetails;
+
+            if (stage.Shader == ResourceId.Null || shaderDetails == null) return;
+
+            var files = new Dictionary<string, string>();
+
+            // use disassembly for now. It's not compilable GLSL but it's better than
+            // starting with a blank canvas
+            files.Add("Disassembly", shaderDetails.Disassembly);
+
+            VulkanPipelineStateViewer pipeviewer = this;
+
+            ShaderViewer sv = new ShaderViewer(m_Core, false, "main", files,
+
+            // Save Callback
+            (ShaderViewer viewer, Dictionary<string, string> updatedfiles) =>
+            {
+                string compileSource = "";
+                foreach (var kv in updatedfiles)
+                    compileSource += kv.Value;
+
+                // invoke off to the ReplayRenderer to replace the log's shader
+                // with our edited one
+                m_Core.Renderer.BeginInvoke((ReplayRenderer r) =>
+                {
+                    string errs = "";
+
+                    ResourceId from = stage.Shader;
+                    ResourceId to = r.BuildTargetShader("main", compileSource, shaderDetails.DebugInfo.compileFlags, stage.stage, out errs);
+
+                    viewer.BeginInvoke((MethodInvoker)delegate { viewer.ShowErrors(errs); });
+                    if (to == ResourceId.Null)
+                    {
+                        r.RemoveReplacement(from);
+                        pipeviewer.BeginInvoke((MethodInvoker)delegate { m_Core.RefreshStatus(); });
+                    }
+                    else
+                    {
+                        r.ReplaceResource(from, to);
+                        pipeviewer.BeginInvoke((MethodInvoker)delegate { m_Core.RefreshStatus(); });
+                    }
+                });
+            },
+
+            // Close Callback
+            () =>
+            {
+                // remove the replacement on close (we could make this more sophisticated if there
+                // was a place to control replaced resources/shaders).
+                m_Core.Renderer.BeginInvoke((ReplayRenderer r) =>
+                {
+                    r.RemoveReplacement(stage.Shader);
+                    pipeviewer.BeginInvoke((MethodInvoker)delegate { m_Core.RefreshStatus(); });
+                });
+            });
+
+            sv.Show(m_DockContent.DockPanel);
         }
 
         private void ShowCBuffer(VulkanPipelineState.ShaderStage stage, CBufferTag tag)