Improve handling of different cases registering vulkan layer on linux

* In particular, if there is a mismatched layer registered under e.g. /etc then we always need to elevate, even if the user wants their layer to be fixed and registered user-local.
2026-05-29 21:30:53 +00:00 · 2019-10-16 15:12:38 +01:00
parent 6562185e1a
commit 7533c6b7ac
3 changed files with 172 additions and 46 deletions
@@ -340,15 +340,23 @@ void CaptureDialog::vulkanLayerWarn_mouseClick()
  const bool hasOtherJSON = bool(info.flags & VulkanLayerFlags::OtherInstallsRegistered);
  const bool thisRegistered = bool(info.flags & VulkanLayerFlags::ThisInstallRegistered);
  const bool needElevation = bool(info.flags & VulkanLayerFlags::NeedElevation);
-  const bool couldElevate = bool(info.flags & VulkanLayerFlags::CouldElevate);
+  const bool userRegisterable = bool(info.flags & VulkanLayerFlags::UserRegisterable);
  const bool registerAll = bool(info.flags & VulkanLayerFlags::RegisterAll);
  const bool updateAllowed = bool(info.flags & VulkanLayerFlags::UpdateAllowed);

  if(info.flags & VulkanLayerFlags::Unfixable)
  {
    QString msg =
-        tr("There is an unfixable problem with your vulkan layer configuration. Please consult the "
-           "RenderDoc documentation, or package/distribution documentation on linux\n\n");
+        tr("There is an unfixable problem with your vulkan layer configuration.\n\n"
+           "This is most commonly caused by having a distribution-provided package of RenderDoc "
+           "installed, which cannot be modified by another build of RenderDoc.\n\n"
+           "Please consult the RenderDoc documentation, or package/distribution documentation on "
+           "linux. ");
+
+    if(info.otherJSONs.size() > 1)
+      msg += tr("Conflicting manifests:\n\n");
+    else
+      msg += tr("Conflicting manifest:\n\n");

    for(const rdcstr &j : info.otherJSONs)
      msg += j + lit("\n");
@@ -364,12 +372,12 @@ void CaptureDialog::vulkanLayerWarn_mouseClick()
  {
    if(info.otherJSONs.size() > 1)
      msg +=
-          tr("there are other RenderDoc builds registered already. They must be disabled so that "
-             "capture can happen without nasty clashes.");
+          tr("there are other conflicting RenderDoc builds registered already. They must be "
+             "disabled so that vulkan programs can be captured without crashes.");
    else
      msg +=
-          tr("there is another RenderDoc build registered already. It must be disabled so that "
-             "capture can happen without nasty clashes.");
+          tr("there is another conflicting RenderDoc build registered already. It must be disabled "
+             "so that vulkan programs can be captured without crashes.");

    if(!thisRegistered)
      msg += tr(" Also ");
@@ -410,44 +418,69 @@ void CaptureDialog::vulkanLayerWarn_mouseClick()
    msg += lit("\n");
  }

+  if(needElevation)
+    msg +=
+        tr("Due to some builds being in privileged locations, RenderDoc must elevate permissions "
+           "to update them.\n\n");
+
  msg += tr("This is a one-off change, it won't be needed again unless the installation moves.");

  QMessageBox::StandardButton install = RDDialog::question(this, caption, msg, RDDialog::YesNoCancel);

  if(install == QMessageBox::Yes)
  {
-    bool run = false;
-    bool admin = false;
+    bool admin = needElevation;
+    bool system = true;    // default to system-wide install
+    bool run = true;       // default to running

-    // if we need to elevate, just try it.
-    if(needElevation)
+    // if we could install user-local, ask the user if that's what they want.
+    if(userRegisterable)
    {
-      admin = run = true;
-    }
-    // if we could elevate, ask the user
-    else if(couldElevate)
-    {
-      QMessageBox::StandardButton elevate = RDDialog::question(
-          this, tr("System layer install"),
-          tr("Do you want to elevate permissions to install the layer at a system level?\n\n"
-             "If you click 'No', the layer will be installed at a per-user level."),
-          RDDialog::YesNoCancel);
+      msg =
+          tr("Do you want to install the layer at a system level?\n\n"
+             "If you click 'No', the layer will be installed at a per-user level.");
+
+      if(needElevation)
+        msg +=
+            tr("\n\nNote that RenderDoc needs to elevate permissions to update the registration "
+               "regardless.");
+      else
+        msg +=
+            tr("\n\nNote that RenderDoc will need to elevate permissions to register at system "
+               "level.");
+
+      QMessageBox::StandardButton elevate =
+          RDDialog::question(this, tr("Install at system level"), msg, RDDialog::YesNoCancel);

      if(elevate == QMessageBox::Yes)
-        admin = true;
+        admin = system = true;
      else if(elevate == QMessageBox::No)
-        admin = false;
+        system = false;

      run = (elevate != QMessageBox::Cancel);
    }
-    // otherwise run non-elevated
-    else
-    {
-      run = true;
-    }

    if(run)
    {
+      auto regComplete = [this, admin]() {
+        bool needReg = RENDERDOC_NeedVulkanLayerRegistration(NULL);
+        ui->vulkanLayerWarn->setVisible(needReg);
+
+#if !defined(Q_OS_LINUX)
+        // can't alert the user on linux because the command might still be running - there's
+        // seemingly no portable way to wait for the command to finish.
+        if(needReg)
+        {
+          QString err = tr("Vulkan layer registration failed for unknown reasons.");
+
+          if(admin)
+            err += tr(" Ensure that the elevation to admin permissions succeeded.");
+
+          RDDialog::critical(this, tr("Layer registration failed"), err);
+        }
+#endif
+      };
+
      if(admin)
      {
 // linux sometimes can't run GUI apps as root, so we have to run renderdoccmd. Check that it's
@@ -480,25 +513,40 @@ void CaptureDialog::vulkanLayerWarn_mouseClick()
          // it's in the path, we can continue
        }

-        RunProcessAsAdmin(cmd, QStringList() << lit("vulkanregister") << lit("--system"), this,
-                          true, [this]() { ui->vulkanLayerWarn->setVisible(false); });
+        QStringList renderdoccmdParams;
+
+        renderdoccmdParams << lit("vulkanregister");
+        if(system)
+          renderdoccmdParams << lit("--system");
+
+        if(!RunProcessAsAdmin(cmd, renderdoccmdParams, this, true, regComplete))
+          regComplete();
 #else
-        RunProcessAsAdmin(qApp->applicationFilePath(),
-                          QStringList() << lit("--install_vulkan_layer") << lit("root"), this,
-                          false, [this]() { ui->vulkanLayerWarn->setVisible(false); });
+        QStringList qrenderdocParams;
+
+        qrenderdocParams << lit("--install_vulkan_layer");
+        if(system)
+          qrenderdocParams << lit("root");
+
+        if(!RunProcessAsAdmin(qApp->applicationFilePath(), qrenderdocParams, this, false, regComplete))
+          regComplete();
 #endif
        return;
      }
      else
      {
        QProcess *process = new QProcess;
-        process->start(qApp->applicationFilePath(), QStringList() << lit("--install_vulkan_layer")
-                                                                  << lit("user"));
+        process->start(qApp->applicationFilePath(), QStringList()
+                                                        << lit("--install_vulkan_layer")
+                                                        << (system ? lit("root") : lit("user")));
        process->waitForFinished(300);

        // when the process exits, delete
        QObject::connect(process, OverloadedSlot<int, QProcess::ExitStatus>::of(&QProcess::finished),
-                         [process](int exitCode, QProcess::ExitStatus) { process->deleteLater(); });
+                         [regComplete, process](int exitCode, QProcess::ExitStatus) {
+                           process->deleteLater();
+                           regComplete();
+                         });
      }
    }

@@ -3926,10 +3926,18 @@ DOCUMENT(R"(A set of flags giving details of the current status of vulkan layer

  Fixing any issues will require elevation to system administrator privileges.

-.. data:: CouldElevate
+.. data:: UserRegisterable

-  Fixing issues could be done purely as a user, but can optionally be done at system level with
-  system administrator privileges.
+  This layer can be registered as user-local, as well as system-wide. If :data:`NeedElevation` isn't
+  also set then the entire process can be done un-elevated if user-local is desired.
+
+  .. note::
+  
+    If the :data:`NeedElevation` flag is set then elevation is required to fix the layer
+    registration, even if a user-local registration is desired.
+
+    Most commonly this situation arises if there is no other registration, or the existing one is
+    already user-local.

 .. data:: RegisterAll

@@ -3951,7 +3959,7 @@ enum class VulkanLayerFlags : uint32_t
  OtherInstallsRegistered = 0x1,
  ThisInstallRegistered = 0x2,
  NeedElevation = 0x4,
-  CouldElevate = 0x8,
+  UserRegisterable = 0x8,
  RegisterAll = 0x10,
  UpdateAllowed = 0x20,
  Unfixable = 0x40,
@@ -457,7 +457,7 @@ bool VulkanReplay::CheckVulkanLayer(VulkanLayerFlags &flags, std::vector<std::st
      numMatch++;
  }

-  flags = VulkanLayerFlags::CouldElevate | VulkanLayerFlags::UpdateAllowed;
+  flags = VulkanLayerFlags::UserRegisterable | VulkanLayerFlags::UpdateAllowed;

  if(numMatch >= 1)
    flags |= VulkanLayerFlags::ThisInstallRegistered;
@@ -466,11 +466,23 @@ bool VulkanReplay::CheckVulkanLayer(VulkanLayerFlags &flags, std::vector<std::st
  if(numExist == 1 && numMatch == 1)
    return false;

+  if(numMatch == 1 && exist[(int)LayerPath::etc] && match[(int)LayerPath::etc])
+  {
+    // if only /etc is registered matching us, keep things simple and don't allow unregistering it
+    // and registering the /home. Just unregister the /home that doesn't match
+    flags &= ~(VulkanLayerFlags::UserRegisterable | VulkanLayerFlags::UpdateAllowed);
+  }
+
  if(exist[(int)LayerPath::usr] && !match[(int)LayerPath::usr])
    otherJSONs.push_back(LayerRegistrationPath(LayerPath::usr));

  if(exist[(int)LayerPath::etc] && !match[(int)LayerPath::etc])
+  {
+    // if the /etc manifest doesn't match we need to elevate to fix it regardless of whether we
+    // delete it in favour of a /home manifest, or if we update it.
+    flags |= VulkanLayerFlags::NeedElevation;
    otherJSONs.push_back(LayerRegistrationPath(LayerPath::etc));
+  }

  if(exist[(int)LayerPath::home] && !match[(int)LayerPath::home])
    otherJSONs.push_back(LayerRegistrationPath(LayerPath::home));
@@ -480,12 +492,40 @@ bool VulkanReplay::CheckVulkanLayer(VulkanLayerFlags &flags, std::vector<std::st

  if(exist[(int)LayerPath::usr] && match[(int)LayerPath::usr])
  {
-    // just need to unregister others
+    // just need to unregister others, but we can't user-local register anymore (as that would
+    // require removing the one in /usr which we can't do)
+    flags &= ~VulkanLayerFlags::UserRegisterable;
+
+    // any other manifests that exist, even if they match, are considered others.
+    if(exist[(int)LayerPath::home])
+    {
+      otherJSONs.push_back(LayerRegistrationPath(LayerPath::home));
+      flags |= VulkanLayerFlags::OtherInstallsRegistered;
+    }
+
+    // any other manifests that exist, even if they match, are considered others.
+    if(exist[(int)LayerPath::etc])
+    {
+      otherJSONs.push_back(LayerRegistrationPath(LayerPath::etc));
+      flags |= VulkanLayerFlags::OtherInstallsRegistered | VulkanLayerFlags::NeedElevation;
+    }
  }
  else
  {
-    myJSONs.push_back(LayerRegistrationPath(LayerPath::etc));
-    myJSONs.push_back(LayerRegistrationPath(LayerPath::home));
+    // if we have multiple matches but they are all correct, and there are no other JSONs we just
+    // report that home needs to be unregistered.
+    if(otherJSONs.empty() && exist[(int)LayerPath::etc] && match[(int)LayerPath::etc])
+    {
+      flags &= ~(VulkanLayerFlags::UserRegisterable | VulkanLayerFlags::UpdateAllowed);
+      flags |= VulkanLayerFlags::OtherInstallsRegistered;
+      myJSONs.push_back(LayerRegistrationPath(LayerPath::etc));
+      otherJSONs.push_back(LayerRegistrationPath(LayerPath::home));
+    }
+    else
+    {
+      myJSONs.push_back(LayerRegistrationPath(LayerPath::etc));
+      myJSONs.push_back(LayerRegistrationPath(LayerPath::home));
+    }
  }

  if(exist[(int)LayerPath::usr] && !match[(int)LayerPath::usr])
@@ -500,7 +540,39 @@ bool VulkanReplay::CheckVulkanLayer(VulkanLayerFlags &flags, std::vector<std::st

 void VulkanReplay::InstallVulkanLayer(bool systemLevel)
 {
+  std::string usrPath = LayerRegistrationPath(LayerPath::usr);
  std::string homePath = LayerRegistrationPath(LayerPath::home);
+  std::string etcPath = LayerRegistrationPath(LayerPath::etc);
+
+  if(FileExists(usrPath))
+  {
+    // if the usr path exists, all we can do is try to remove etc & home. This assumes a
+    // system-level install
+    if(!systemLevel)
+    {
+      RDCERR("Can't register user-local with manifest under /usr");
+      return;
+    }
+
+    if(FileExists(homePath))
+    {
+      if(unlink(homePath.c_str()) < 0)
+      {
+        const char *const errtext = strerror(errno);
+        RDCERR("Error removing %s: %s", homePath.c_str(), errtext);
+      }
+    }
+    if(FileExists(etcPath))
+    {
+      if(unlink(etcPath.c_str()) < 0)
+      {
+        const char *const errtext = strerror(errno);
+        RDCERR("Error removing %s: %s", etcPath.c_str(), errtext);
+      }
+    }
+
+    return;
+  }

  // if we want to install to the system and there's a registration in $HOME, delete it
  if(systemLevel && FileExists(homePath))
@@ -512,8 +584,6 @@ void VulkanReplay::InstallVulkanLayer(bool systemLevel)
    }
  }

-  std::string etcPath = LayerRegistrationPath(LayerPath::etc);
-
  // and vice-versa
  if(!systemLevel && FileExists(etcPath))
  {