Allow using shader processing tools for custom shaders

* This is primarily useful for HLSL on Vulkan, but could be used with any other
  combination. If multiple tools can perform the conversion, the highest
  priority one is used.

renderdoc/api/replay/pipestate.h

@@ -184,15 +184,6 @@ requirements.
 )");
   rdcstr OutputAbbrev() const;
 
-  DOCUMENT(R"(Retrieves the common file extension for high level shaders in the current API.
-
-Typically this is ``glsl`` or ``hlsl``.
-
-:return: The file extension with no ``.``.
-:rtype: ``str``
-)");
-  rdcstr GetShaderExtension() const;
-
   DOCUMENT(R"(Retrieves the viewport for a given index.
 
 :param int index: The index to retrieve.

renderdoc/api/replay/pipestate.inl

@@ -166,16 +166,6 @@ const VKPipe::Shader &PipeState::GetVulkanStage(ShaderStage stage) const
   return m_Vulkan->computeShader;
 }
 
-rdcstr PipeState::GetShaderExtension() const
-{
-  if(IsCaptureGL() || IsCaptureVK())
-  {
-    return "glsl";
-  }
-
-  return "hlsl";
-}
-
 Viewport PipeState::GetViewport(int index) const
 {
   Viewport ret = {};

renderdoc/api/replay/renderdoc_replay.h

@@ -982,19 +982,19 @@ or hardware-specific ISA formats.
 
   DOCUMENT(R"(Builds a shader suitable for running on the local replay instance as a custom shader.
 
-The language used is native to the local renderer - HLSL for D3D based renderers, GLSL otherwise.
-
 See :data:`TextureDisplay.customShaderId`.
 
 :param str entry: The entry point to use when compiling.
-:param str source: The source file.
+:param ShaderEncoding sourceEncoding: The encoding of the source data.
+:param bytes source: The source data itself.
 :param int compileFlags: API-specific compilation flags.
 :param ShaderStage type: The stage that this shader will be executed at.
 :return: A ``tuple`` with the id of the new shader if compilation was successful,
   :meth:`ResourceId.Null` otherwise, and a ``str`` with any warnings/errors from compilation.
 :rtype: ``tuple`` of :class:`ResourceId` and ``str``.
 )");
-  virtual rdcpair<ResourceId, rdcstr> BuildCustomShader(const char *entry, const char *source,
+  virtual rdcpair<ResourceId, rdcstr> BuildCustomShader(const char *entry,
+                                                        ShaderEncoding sourceEncoding, bytebuf source,
                                                         const ShaderCompileFlags &compileFlags,
                                                         ShaderStage type) = 0;
 
@@ -1008,8 +1008,6 @@ See :meth:`BuildCustomShader`.
 
   DOCUMENT(R"(Builds a shader suitable for running in the capture's API as a replacement shader.
 
-The language used is native to the API's renderer - HLSL for D3D based renderers, GLSL otherwise.
-
 :param str entry: The entry point to use when compiling.
 :param ShaderEncoding sourceEncoding: The encoding of the source data.
 :param bytes source: The source data itself.