Expose vulkan layer registration in renderdoccmd on win32. Closes #1690

* This is needed so that the functional tests can elevate and run renderdoccmd
  to register the vulkan layer, if needed.
* At the same time remove the old spammy message and ignore flag - this dates
  back to before the UI existed, and that should be the way users run RenderDoc
  generally and it has a good UI for walking through layer registration if
  needed.
* The command is always available, but will only show up in help if attention is
  needed.
* Also fix registering installs on shared drives.

qrenderdoc/Windows/Dialogs/CaptureDialog.cpp

@@ -515,9 +515,11 @@ void CaptureDialog::vulkanLayerWarn_mouseClick()
 
         QStringList renderdoccmdParams;
 
-        renderdoccmdParams << lit("vulkanregister");
+        renderdoccmdParams << lit("vulkanlayer");
         if(system)
           renderdoccmdParams << lit("--system");
+        else
+          renderdoccmdParams << lit("--user");
 
         if(!RunProcessAsAdmin(cmd, renderdoccmdParams, this, true, regComplete))
           regComplete();

renderdoc/api/replay/replay_enums.h

@@ -4126,6 +4126,10 @@ DOCUMENT(R"(A set of flags giving details of the current status of vulkan layer
 .. data:: Unfixable
 
   The current situation is not fixable automatically and requires user intervention/disambiguation.
+
+.. data:: Unsupported
+
+  Vulkan is not supported by this build of RenderDoc and the layer cannot be registered.
 )");
 enum class VulkanLayerFlags : uint32_t
 {
@@ -4137,6 +4141,7 @@ enum class VulkanLayerFlags : uint32_t
   RegisterAll = 0x10,
   UpdateAllowed = 0x20,
   Unfixable = 0x40,
+  Unsupported = 0x80,
 };
 
 BITMASK_OPERATORS(VulkanLayerFlags);

renderdoc/core/core.h

@@ -475,6 +475,8 @@ public:
     if(m_VulkanCheck)
       return m_VulkanCheck(flags, myJSONs, otherJSONs);
 
+    flags = VulkanLayerFlags::Unfixable | VulkanLayerFlags::Unsupported;
+
     return false;
   }
 

renderdoc/driver/vulkan/vk_posix.cpp

@@ -29,6 +29,8 @@
 #include "vk_replay.h"
 
 #include <errno.h>
+#include <limits.h>
+#include <stdlib.h>
 #include <sys/stat.h>
 #include <unistd.h>
 
@@ -337,6 +339,14 @@ static rdcstr GetSOFromJSON(const rdcstr &json)
 
   delete[] json_string;
 
+  // get the realpath, if this is a real filename
+  char *resolved = realpath(ret.c_str(), NULL);
+  if(resolved && resolved[0])
+  {
+    ret = resolved;
+    free(resolved);
+  }
+
   return ret;
 }
 
@@ -391,17 +401,6 @@ void MakeParentDirs(rdcstr file)
 bool VulkanReplay::CheckVulkanLayer(VulkanLayerFlags &flags, rdcarray<rdcstr> &myJSONs,
                                     rdcarray<rdcstr> &otherJSONs)
 {
-  // see if the user has suppressed all this checking as a "I know what I'm doing" measure
-
-  const char *home_path = getenv("HOME");
-  if(home_path == NULL)
-    home_path = "";
-  if(FileExists(rdcstr(home_path) + "/.renderdoc/ignore_vulkan_layer_issues"))
-  {
-    flags = VulkanLayerFlags::ThisInstallRegistered;
-    return false;
-  }
-
   ////////////////////////////////////////////////////////////////////////////////////////
   // check that there's only one layer registered, and it points to the same .so file that
   // we are running with in this instance of renderdoccmd
@@ -409,6 +408,13 @@ bool VulkanReplay::CheckVulkanLayer(VulkanLayerFlags &flags, rdcarray<rdcstr> &m
   rdcstr librenderdoc_path;
   FileIO::GetLibraryFilename(librenderdoc_path);
 
+  char *resolved = realpath(librenderdoc_path.c_str(), NULL);
+  if(resolved && resolved[0])
+  {
+    librenderdoc_path = resolved;
+    free(resolved);
+  }
+
   if(librenderdoc_path.empty() || !FileExists(librenderdoc_path))
   {
     RDCERR("Couldn't determine current library path!");

renderdoccmd/renderdoccmd.cpp

@@ -38,6 +38,8 @@ std::string conv(const rdcstr &s)
   return std::string(s.begin(), s.end());
 }
 
+static int command_usage(std::string command = "");
+
 // normally this is in the renderdoc core library, but it's needed for the 'unknown enum' path,
 // so we implement it here using ostringstream. It's not great, but this is a very uncommon path -
 // either for invalid values or for when a new enum is added and the code isn't updated
@@ -122,64 +124,6 @@ void DisplayRendererPreview(IReplayController *renderer, uint32_t width, uint32_
   DisplayRendererPreview(renderer, d, width, height, numLoops);
 }
 
-std::map<std::string, Command *> commands;
-std::map<std::string, std::string> aliases;
-
-void add_command(const std::string &name, Command *cmd)
-{
-  commands[name] = cmd;
-}
-
-void add_alias(const std::string &alias, const std::string &command)
-{
-  aliases[alias] = command;
-}
-
-static void clean_up()
-{
-  for(auto it = commands.begin(); it != commands.end(); ++it)
-    delete it->second;
-}
-
-static int command_usage(std::string command = "")
-{
-  if(!command.empty())
-    std::cerr << command << " is not a valid command." << std::endl << std::endl;
-
-  std::cerr << "Usage: renderdoccmd <command> [args ...]" << std::endl;
-  std::cerr << "Command line tool for capture & replay with RenderDoc." << std::endl << std::endl;
-
-  std::cerr << "Command can be one of:" << std::endl;
-
-  size_t max_width = 0;
-  for(auto it = commands.begin(); it != commands.end(); ++it)
-  {
-    if(it->second->IsInternalOnly())
-      continue;
-
-    max_width = std::max(max_width, it->first.length());
-  }
-
-  for(auto it = commands.begin(); it != commands.end(); ++it)
-  {
-    if(it->second->IsInternalOnly())
-      continue;
-
-    std::cerr << "  " << it->first;
-    for(size_t n = it->first.length(); n < max_width + 4; n++)
-      std::cerr << ' ';
-    std::cerr << it->second->Description() << std::endl;
-  }
-  std::cerr << std::endl;
-
-  std::cerr << "To see details of any command, see 'renderdoccmd <command> --help'" << std::endl
-            << std::endl;
-
-  std::cerr << "For more information, see <https://renderdoc.org/>." << std::endl;
-
-  return 2;
-}
-
 static std::vector<std::string> version_lines;
 
 struct VersionCommand : public Command
@@ -1184,13 +1128,268 @@ struct EmbeddedSectionCommand : public Command
   }
 };
 
+struct VulkanRegisterCommand : public Command
+{
+private:
+  bool m_LayerNeedUpdate = false;
+  VulkanLayerRegistrationInfo m_Info;
+
+public:
+  VulkanRegisterCommand(const GlobalEnvironment &env) : Command(env)
+  {
+    m_LayerNeedUpdate = RENDERDOC_NeedVulkanLayerRegistration(&m_Info);
+  }
+  virtual void AddOptions(cmdline::parser &parser)
+  {
+    parser.add("explain", '\0',
+               "Explain what the status of the layer registration is, and how it can be resolved");
+    parser.add("register", '\0', "Register RenderDoc's vulkan layer");
+    parser.add("user", '\0',
+               "Install layer registration at user-local level instead of system-wide");
+    parser.add("system", '\0',
+               "Install layer registration system-wide (requires admin privileges)");
+  }
+  virtual const char *Description() { return "Vulkan layer registration needs attention"; }
+  virtual bool IsInternalOnly()
+  {
+    // if the layer is registered and doesn't need an update, don't report this command in help
+    return !m_LayerNeedUpdate;
+  }
+  virtual bool IsCaptureCommand() { return false; }
+  virtual int Execute(cmdline::parser &parser, const CaptureOptions &)
+  {
+    if(parser.exist("explain") || !parser.exist("register"))
+    {
+      if(m_LayerNeedUpdate)
+      {
+        if(m_Info.flags & VulkanLayerFlags::Unfixable)
+        {
+          std::cerr << "** There is an unfixable problem with your vulkan layer configuration.\n\n"
+                       "This is most commonly caused by having a distribution-provided package of "
+                       "RenderDoc "
+                       "installed, which cannot be modified by another build of RenderDoc.\n\n"
+                       "Please consult the RenderDoc documentation, or package/distribution "
+                       "documentation on "
+                       "linux."
+                    << std::endl;
+
+          if(m_Info.otherJSONs.size() > 1)
+            std::cerr << "Conflicting manifests:\n\n";
+          else
+            std::cerr << "Conflicting manifest:\n\n";
+
+          for(const rdcstr &j : m_Info.otherJSONs)
+            std::cerr << conv(j) << std::endl;
+
+          return 0;
+        }
+
+        std::cerr << "*************************************************************************"
+                  << std::endl;
+        std::cerr << "**          Warning: Vulkan layer not correctly registered.            **"
+                  << std::endl;
+        std::cerr << std::endl;
+
+        if(m_Info.flags & VulkanLayerFlags::OtherInstallsRegistered)
+          std::cerr << " - Non-matching RenderDoc layer(s) are registered." << std::endl;
+
+        if(!(m_Info.flags & VulkanLayerFlags::ThisInstallRegistered))
+          std::cerr << " - This build's RenderDoc layer is not registered." << std::endl;
+
+        std::cerr << std::endl;
+
+        std::cerr << " To fix this, the following actions must take place: " << std::endl
+                  << std::endl;
+
+        const bool registerAll = bool(m_Info.flags & VulkanLayerFlags::RegisterAll);
+        const bool updateAllowed = bool(m_Info.flags & VulkanLayerFlags::UpdateAllowed);
+
+        for(const rdcstr &j : m_Info.otherJSONs)
+          std::cerr << (updateAllowed ? " Unregister/update: " : " Unregister: ") << j.c_str()
+                    << std::endl;
+
+        if(!(m_Info.flags & VulkanLayerFlags::ThisInstallRegistered))
+        {
+          if(registerAll)
+          {
+            for(const rdcstr &j : m_Info.myJSONs)
+              std::cerr << (updateAllowed ? " Register/update: " : " Register: ") << j.c_str()
+                        << std::endl;
+          }
+          else
+          {
+            std::cerr << (updateAllowed ? " Register one of:" : " Register/update one of:")
+                      << std::endl;
+            for(const rdcstr &j : m_Info.myJSONs)
+              std::cerr << "  -- " << j.c_str() << "\n";
+          }
+        }
+
+        std::cerr << std::endl;
+
+        if(m_Info.flags & VulkanLayerFlags::UserRegisterable)
+        {
+          std::cerr << " You must choose whether to register at user or system level." << std::endl
+                    << std::endl;
+          std::cerr
+              << " 'vulkanlayer --register --user' will register the layer local to your user."
+              << std::endl;
+          if(m_Info.flags & VulkanLayerFlags::NeedElevation)
+            std::cerr << "  (This requires admin permissions to unregister other installs)"
+                      << std::endl;
+          else
+            std::cerr << " (This does not require admin permission)" << std::endl;
+          std::cerr << std::endl;
+          std::cerr << " If you want to install system-wide, run 'vulkanlayer --system'."
+                    << std::endl;
+          std::cerr << "  (This requires admin permission)" << std::endl;
+
+          std::cerr << "*************************************************************************"
+                    << std::endl;
+          std::cerr << std::endl;
+        }
+        else
+        {
+          std::cerr << " The layer must be registered at system level, this operation requires\n"
+                    << " admin permissions." << std::endl;
+          std::cerr << std::endl;
+          std::cerr << " Run 'vulkanlayer --register --system' as administrator to register."
+                    << std::endl;
+
+          std::cerr << std::endl;
+          std::cerr << "*************************************************************************"
+                    << std::endl;
+          std::cerr << std::endl;
+        }
+      }
+      else
+      {
+        std::cerr << "The RenderDoc vulkan layer appears to be correctly registered." << std::endl;
+      }
+
+      // don't do anything if we're just explaining the situation
+      return 0;
+    }
+
+    bool user = parser.exist("user"), system = parser.exist("system");
+
+    if(m_Info.flags & VulkanLayerFlags::UserRegisterable)
+    {
+      if(user)
+      {
+        std::cerr << "Vulkan layer cannot be registered at user level." << std::endl;
+        return 1;
+      }
+    }
+
+    if(user && system)
+    {
+      std::cerr << "Vulkan layer cannot be registered at user and system levels." << std::endl;
+      return 1;
+    }
+    else if(user || system)
+    {
+      RENDERDOC_UpdateVulkanLayerRegistration(system);
+
+      if(RENDERDOC_NeedVulkanLayerRegistration(NULL))
+      {
+        std::cerr << "Vulkan layer registration not successful. ";
+        if(system)
+          std::cerr << "Check that you are running as administrator";
+        std::cerr << std::endl;
+      }
+    }
+    else
+    {
+      std::cerr << "You must select either '--user' or '--system' to choose where to register the "
+                   "vulkan layer."
+                << std::endl;
+      return 1;
+    }
+
+    return 0;
+  }
+};
+
 REPLAY_PROGRAM_MARKER()
 
+VulkanRegisterCommand *vulkan = NULL;
+
+std::map<std::string, Command *> commands;
+std::map<std::string, std::string> aliases;
+
+void add_command(const std::string &name, Command *cmd)
+{
+  commands[name] = cmd;
+}
+
+void add_alias(const std::string &alias, const std::string &command)
+{
+  aliases[alias] = command;
+}
+
+static void clean_up()
+{
+  for(auto it = commands.begin(); it != commands.end(); ++it)
+    delete it->second;
+}
+
+static int command_usage(std::string command)
+{
+  if(!command.empty())
+    std::cerr << command << " is not a valid command." << std::endl << std::endl;
+
+  if(vulkan && !vulkan->IsInternalOnly())
+    std::cerr << "** NOTE: Vulkan layer registration problem detected.\n"
+                 "** Run 'vulkanlayer --explain' for more details"
+              << std::endl
+              << std::endl;
+
+  std::cerr << "Usage: renderdoccmd <command> [args ...]" << std::endl;
+  std::cerr << "Command line tool for capture & replay with RenderDoc." << std::endl << std::endl;
+
+  std::cerr << "Command can be one of:" << std::endl;
+
+  size_t max_width = 0;
+  for(auto it = commands.begin(); it != commands.end(); ++it)
+  {
+    if(it->second->IsInternalOnly())
+      continue;
+
+    max_width = std::max(max_width, it->first.length());
+  }
+
+  for(auto it = commands.begin(); it != commands.end(); ++it)
+  {
+    if(it->second->IsInternalOnly())
+      continue;
+
+    std::cerr << "  " << it->first;
+    for(size_t n = it->first.length(); n < max_width + 4; n++)
+      std::cerr << ' ';
+    std::cerr << it->second->Description() << std::endl;
+  }
+  std::cerr << std::endl;
+
+  std::cerr << "To see details of any command, see 'renderdoccmd <command> --help'" << std::endl
+            << std::endl;
+
+  std::cerr << "For more information, see <https://renderdoc.org/>." << std::endl;
+
+  return 2;
+}
+
 int renderdoccmd(GlobalEnvironment &env, std::vector<std::string> &argv)
 {
   // we don't need this in renderdoccmd.
   env.enumerateGPUs = false;
 
+  vulkan = new VulkanRegisterCommand(env);
+
+  // if vulkan isn't supported, or the layer is fully registered, this command will not be listed
+  // in help so it will be invisible
+  add_command("vulkanlayer", vulkan);
+
   try
   {
     // add basic commands, and common aliases

renderdoccmd/renderdoccmd_linux.cpp

@@ -48,153 +48,6 @@ void Daemonise()
   daemon(1, 0);
 }
 
-struct VulkanRegisterCommand : public Command
-{
-  VulkanRegisterCommand(const GlobalEnvironment &env) : Command(env) {}
-  virtual void AddOptions(cmdline::parser &parser)
-  {
-    parser.add("ignore", 'i', "Do nothing and don't warn about Vulkan layer issues.");
-    parser.add(
-        "system", '\0',
-        "Install layer registration to /etc instead of $HOME/.local (requires root privileges)");
-  }
-  virtual const char *Description()
-  {
-    return "Attempt to automatically fix Vulkan layer registration issues";
-  }
-  virtual bool IsInternalOnly() { return false; }
-  virtual bool IsCaptureCommand() { return false; }
-  virtual int Execute(cmdline::parser &parser, const CaptureOptions &)
-  {
-    bool ignore = (parser.exist("ignore"));
-
-    if(ignore)
-    {
-      std::cout << "Not fixing vulkan layer issues, and suppressing future warnings." << std::endl;
-      std::cout << "To undo, remove '$HOME/.renderdoc/ignore_vulkan_layer_issues'." << std::endl;
-
-      std::string ignorePath = std::string(getenv("HOME")) + "/.renderdoc/";
-
-      mkdir(ignorePath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
-
-      ignorePath += "ignore_vulkan_layer";
-
-      FILE *f = fopen(ignorePath.c_str(), "w");
-
-      if(f)
-      {
-        fputs("This file suppresses any checks for vulkan layer issues.\n", f);
-        fputs("Delete this file to restore default checking.\n", f);
-
-        fclose(f);
-      }
-      else
-      {
-        std::cerr << "Couldn't create '$HOME/.renderdoc/ignore_vulkan_layer_issues'." << std::endl;
-      }
-
-      return 0;
-    }
-
-    RENDERDOC_UpdateVulkanLayerRegistration(parser.exist("system"));
-
-    return 0;
-  }
-};
-
-void VerifyVulkanLayer(const GlobalEnvironment &env, int argc, char *argv[])
-{
-  VulkanLayerRegistrationInfo info;
-
-  bool needUpdate = RENDERDOC_NeedVulkanLayerRegistration(&info);
-
-  if(!needUpdate)
-  {
-    if(!(info.flags & VulkanLayerFlags::Unfixable))
-      add_command("vulkanregister", new VulkanRegisterCommand(env));
-    return;
-  }
-
-  std::cerr << "*************************************************************************"
-            << std::endl;
-  std::cerr << "**          Warning: Vulkan capture possibly not configured.           **"
-            << std::endl;
-  std::cerr << std::endl;
-
-  if(info.flags & VulkanLayerFlags::OtherInstallsRegistered)
-    std::cerr << "Multiple RenderDoc layers are registered, possibly from different builds."
-              << std::endl;
-
-  if(!(info.flags & VulkanLayerFlags::ThisInstallRegistered))
-    std::cerr << "This build's RenderDoc layer is not registered." << std::endl;
-
-  std::cerr << "To fix this, the following actions must take place: " << std::endl << std::endl;
-
-  const bool registerAll = bool(info.flags & VulkanLayerFlags::RegisterAll);
-  const bool updateAllowed = bool(info.flags & VulkanLayerFlags::UpdateAllowed);
-
-  for(const rdcstr &j : info.otherJSONs)
-    std::cerr << (updateAllowed ? "Unregister/update: " : "Unregister: ") << j.c_str() << std::endl;
-
-  if(!(info.flags & VulkanLayerFlags::ThisInstallRegistered))
-  {
-    if(registerAll)
-    {
-      for(const rdcstr &j : info.myJSONs)
-        std::cerr << (updateAllowed ? "Register/update: " : "Register: ") << j.c_str() << std::endl;
-    }
-    else
-    {
-      std::cerr << (updateAllowed ? "Register one of:" : "Register/update one of:") << std::endl;
-      for(const rdcstr &j : info.myJSONs)
-        std::cerr << "  -- " << j.c_str() << "\n";
-    }
-  }
-
-  std::cerr << std::endl;
-
-  if(info.flags & VulkanLayerFlags::Unfixable)
-  {
-    std::cerr << "NOTE: The renderdoc layer registered in /usr is reserved for distribution"
-              << std::endl;
-    std::cerr << "controlled packages. RenderDoc cannot automatically unregister this even"
-              << std::endl;
-    std::cerr << "with root permissions, you must fix this conflict manually." << std::endl
-              << std::endl;
-
-    std::cerr << "*************************************************************************"
-              << std::endl;
-    std::cerr << std::endl;
-
-    return;
-  }
-
-  std::cerr << "NOTE: Automatically removing or changing the layer registered in /etc" << std::endl;
-  std::cerr << "will require root privileges." << std::endl << std::endl;
-
-  std::cerr << "To fix these issues run the 'vulkanregister' command." << std::endl;
-  std::cerr << "Use 'vulkanregister --help' to see more information." << std::endl;
-  std::cerr << std::endl;
-
-  std::cerr << "By default 'vulkanregister' will register the layer to your $HOME folder."
-            << std::endl;
-  std::cerr << "This does not require root permissions." << std::endl;
-  std::cerr << std::endl;
-  std::cerr << "If you want to install to the system, run 'vulkanregister --system'." << std::endl;
-  std::cerr << "This requires root permissions to write to /etc/vulkan/." << std::endl;
-
-  // just in case there's a strange install that is misdetected or something then allow
-  // users to suppress this message and just say "I know what I'm doing".
-  std::cerr << std::endl;
-  std::cerr << "To suppress this warning in future, run 'vulkanregister --ignore'." << std::endl;
-
-  std::cerr << "*************************************************************************"
-            << std::endl;
-  std::cerr << std::endl;
-
-  add_command("vulkanregister", new VulkanRegisterCommand(env));
-}
-
 static Display *display = NULL;
 
 WindowingData DisplayRemoteServerPreview(bool active, const rdcarray<WindowingSystem> &systems)
@@ -486,10 +339,6 @@ int main(int argc, char *argv[])
   display = env.xlibDisplay = XOpenDisplay(NULL);
 #endif
 
-#if defined(RENDERDOC_SUPPORT_VULKAN)
-  VerifyVulkanLayer(env, argc, argv);
-#endif
-
   // add compiled-in support to version line
   {
     std::string support = "APIs supported at compile-time: ";

util/test/rdtest/runner.py

@@ -12,6 +12,7 @@ import renderdoc as rd
 from . import util
 from . import testcase
 from .logging import log
+from pathlib import Path
 
 
 def get_tests():
@@ -225,6 +226,13 @@ def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests
                 args = sys.argv.copy()
                 args.append("--internal_vulkan_register")
 
+                for i in range(len(args)):
+                    if os.path.exists(args[i]):
+                        args[i] = str(Path(args[i]).resolve())
+
+                if 'renderdoccmd' in sys.executable:
+                    args = ['vulkanlayer', '--register', '--system']
+
                 ctypes.windll.shell32.ShellExecuteW(None, "runas", sys.executable, ' '.join(args), None, 1)
 
                 time.sleep(10)