Make it a bit easier to self-host capture on vulkan

* You need to register rdocself.json by hand but otherwise then the same
  workflow works as for other APIs.

renderdoc/driver/vulkan/CMakeLists.txt

@@ -93,6 +93,7 @@ elseif(ENABLE_GGP)
     add_definitions(-DVK_USE_PLATFORM_GGP)
 
     set(VULKAN_LAYER_MODULE_PATH "${CMAKE_INSTALL_PREFIX}/lib${LIB_SUFFIX}/${LIB_SUBFOLDER_TRAIL_SLASH}librenderdoc.so")
+    set(VULKAN_ENABLE_VAR "ENABLE_RENDERDOC_VULKAN_CAPTURE")
 
     set(json_in ${CMAKE_CURRENT_SOURCE_DIR}/renderdoc.json)
     set(json_out ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/renderdoc_capture.json)
@@ -116,6 +117,7 @@ elseif(UNIX)
     endif()
 
     set(VULKAN_LAYER_MODULE_PATH "${CMAKE_INSTALL_PREFIX}/lib${LIB_SUFFIX}/${LIB_SUBFOLDER_TRAIL_SLASH}librenderdoc.so")
+    set(VULKAN_ENABLE_VAR "ENABLE_RENDERDOC_VULKAN_CAPTURE")
 
     set(json_in ${CMAKE_CURRENT_SOURCE_DIR}/renderdoc.json)
     set(json_out ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/renderdoc_capture.json)

renderdoc/driver/vulkan/renderdoc.json

@@ -34,7 +34,7 @@
       }
     ],
     "enable_environment": {
-      "ENABLE_VULKAN_RENDERDOC_CAPTURE": "1"
+      "@VULKAN_ENABLE_VAR@": "1"
     },
     "disable_environment": {
       "DISABLE_VULKAN_RENDERDOC_CAPTURE_@RENDERDOC_VERSION_MAJOR@_@RENDERDOC_VERSION_MINOR@": "1"

renderdoc/driver/vulkan/renderdoc_vulkan.vcxproj

@@ -198,34 +198,6 @@
     <None Include="renderdoc.json" />
     <None Include="vk_apple.mm" />
   </ItemGroup>
-  <Target Name="_jsonProcess" BeforeTargets="PrepareForBuild">
-    <!-- Read the version.h -->
-    <PropertyGroup>
-      <VersionFile>$([System.IO.File]::ReadAllText('$(SolutionDir)renderdoc/api/replay/version.h').Trim())</VersionFile>
-    </PropertyGroup>
-    <!-- Extract the major and minor lines -->
-    <PropertyGroup>
-      <!-- Find the start of the line -->
-      <MajorVersionStart>$(VersionFile.IndexOf('#define RENDERDOC_VERSION_MAJOR'))</MajorVersionStart>
-      <!-- Take the rest of the file and split it by newline to get an array of lines,
-           Take the first line and split it by space
-           Take the third item. This is #define MAJOR foo
-                                            0     1    2
-      -->
-      <MajorVersion>$(VersionFile.Substring($(MajorVersionStart)).Split('
-')[0].Split(' ')[2])</MajorVersion>
-      <MinorVersionStart>$(VersionFile.IndexOf('#define RENDERDOC_VERSION_MINOR'))</MinorVersionStart>
-      <MinorVersion>$(VersionFile.Substring($(MinorVersionStart)).Split('
-')[0].Split(' ')[2])</MinorVersion>
-    </PropertyGroup>
-    <Message Text="Processing layer JSON file for RenderDoc $(MajorVersion).$(MinorVersion)" Importance="High" />
-    <!-- Read the template json, replace the major/minor versions -->
-    <PropertyGroup>
-      <JSONContents>$([System.IO.File]::ReadAllText('$(ProjectDir)renderdoc.json').Replace('@RENDERDOC_VERSION_MAJOR@', '$(MajorVersion)').Replace('@RENDERDOC_VERSION_MINOR@', '$(MinorVersion)').Replace('@VULKAN_LAYER_MODULE_PATH@', '.\\renderdoc.dll'))</JSONContents>
-    </PropertyGroup>
-    <!-- Write it out to the output directory -->
-    <WriteLinesToFile File="$(OutDir)renderdoc.json" Lines="$(JSONContents)" Overwrite="true" />
-  </Target>
   <ItemGroup>
     <ProjectReference Include="..\shaders\spirv\renderdoc_spirv.vcxproj">
       <Project>{0aae0ad1-371b-4a36-9ed1-80e10e960605}</Project>

renderdoc/driver/vulkan/vk_layer.cpp

@@ -58,6 +58,19 @@ class VulkanHook : LibraryHook
     Process::RegisterEnvironmentModification(EnvironmentModification(
         EnvMod::Set, EnvSep::NoSep, "ENABLE_VULKAN_RENDERDOC_CAPTURE", "1"));
 
+#if ENABLED(RDOC_WIN32)
+    // on windows support self-hosted capture by checking our filename and tweaking the env var we
+    // set
+    std::string module_name;
+    FileIO::GetLibraryFilename(module_name);
+    module_name = strupper(get_basename(module_name.substr(0, module_name.rfind('.'))));
+    if(module_name != "RENDERDOC")
+    {
+      Process::RegisterEnvironmentModification(EnvironmentModification(
+          EnvMod::Set, EnvSep::NoSep, ("ENABLE_VULKAN_" + module_name + "_CAPTURE").c_str(), "1"));
+    }
+#endif
+
     // check options to set further variables, and apply
     OptionsUpdated();
   }

renderdoc/renderdoc.vcxproj

@@ -594,6 +594,35 @@
   <ItemGroup>
     <Natvis Include="renderdoc.natvis" />
   </ItemGroup>
+  <Target Name="_jsonProcess" BeforeTargets="PrepareForBuild">
+    <!-- Read the version.h -->
+    <PropertyGroup>
+      <VersionFile>$([System.IO.File]::ReadAllText('$(SolutionDir)renderdoc/api/replay/version.h').Trim())</VersionFile>
+    </PropertyGroup>
+    <!-- Extract the major and minor lines -->
+    <PropertyGroup>
+      <!-- Find the start of the line -->
+      <MajorVersionStart>$(VersionFile.IndexOf('#define RENDERDOC_VERSION_MAJOR'))</MajorVersionStart>
+      <!-- Take the rest of the file and split it by newline to get an array of lines,
+           Take the first line and split it by space
+           Take the third item. This is #define MAJOR foo
+                                            0     1    2
+      -->
+      <MajorVersion>$(VersionFile.Substring($(MajorVersionStart)).Split('
+')[0].Split(' ')[2])</MajorVersion>
+      <MinorVersionStart>$(VersionFile.IndexOf('#define RENDERDOC_VERSION_MINOR'))</MinorVersionStart>
+      <MinorVersion>$(VersionFile.Substring($(MinorVersionStart)).Split('
+')[0].Split(' ')[2])</MinorVersion>
+    </PropertyGroup>
+    <Message Text="Processing layer JSON file for RenderDoc $(MajorVersion).$(MinorVersion)" Importance="High" />
+    <!-- Read the template json, replace the major/minor versions -->
+    <PropertyGroup>
+      <VulkanEnableVar>ENABLE_VULKAN_$(ProjectName)_CAPTURE</VulkanEnableVar>
+      <JSONContents>$([System.IO.File]::ReadAllText('$(SolutionDir)renderdoc/driver/vulkan/renderdoc.json').Replace('@RENDERDOC_VERSION_MAJOR@', '$(MajorVersion)').Replace('@RENDERDOC_VERSION_MINOR@', '$(MinorVersion)').Replace('@VULKAN_LAYER_MODULE_PATH@', '.\\$(ProjectName).dll').Replace('@VULKAN_ENABLE_VAR@', $(VulkanEnableVar.ToUpper())))</JSONContents>
+    </PropertyGroup>
+    <!-- Write it out to the output directory -->
+    <WriteLinesToFile File="$(OutDir)$(ProjectName).json" Lines="$(JSONContents)" Overwrite="true" />
+  </Target>
   <ItemGroup>
     <ProjectReference Include="driver\d3d11\renderdoc_d3d11.vcxproj">
       <Project>{f1e59a05-60d4-4927-9e57-dd191eae90ef}</Project>