From 7533c6b7ac7cac7cfab2d1a1ddc011c693202a47 Mon Sep 17 00:00:00 2001 From: baldurk Date: Wed, 16 Oct 2019 15:12:38 +0100 Subject: [PATCH] Improve handling of different cases registering vulkan layer on linux * In particular, if there is a mismatched layer registered under e.g. /etc then we always need to elevate, even if the user wants their layer to be fixed and registered user-local. --- qrenderdoc/Windows/Dialogs/CaptureDialog.cpp | 120 +++++++++++++------ renderdoc/api/replay/replay_enums.h | 16 ++- renderdoc/driver/vulkan/vk_posix.cpp | 82 ++++++++++++- 3 files changed, 172 insertions(+), 46 deletions(-) diff --git a/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp b/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp index 358a01734..91a1224ae 100644 --- a/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp +++ b/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp @@ -340,15 +340,23 @@ void CaptureDialog::vulkanLayerWarn_mouseClick() const bool hasOtherJSON = bool(info.flags & VulkanLayerFlags::OtherInstallsRegistered); const bool thisRegistered = bool(info.flags & VulkanLayerFlags::ThisInstallRegistered); const bool needElevation = bool(info.flags & VulkanLayerFlags::NeedElevation); - const bool couldElevate = bool(info.flags & VulkanLayerFlags::CouldElevate); + const bool userRegisterable = bool(info.flags & VulkanLayerFlags::UserRegisterable); const bool registerAll = bool(info.flags & VulkanLayerFlags::RegisterAll); const bool updateAllowed = bool(info.flags & VulkanLayerFlags::UpdateAllowed); if(info.flags & VulkanLayerFlags::Unfixable) { QString msg = - tr("There is an unfixable problem with your vulkan layer configuration. Please consult the " - "RenderDoc documentation, or package/distribution documentation on linux\n\n"); + tr("There is an unfixable problem with your vulkan layer configuration.\n\n" + "This is most commonly caused by having a distribution-provided package of RenderDoc " + "installed, which cannot be modified by another build of RenderDoc.\n\n" + "Please consult the RenderDoc documentation, or package/distribution documentation on " + "linux. "); + + if(info.otherJSONs.size() > 1) + msg += tr("Conflicting manifests:\n\n"); + else + msg += tr("Conflicting manifest:\n\n"); for(const rdcstr &j : info.otherJSONs) msg += j + lit("\n"); @@ -364,12 +372,12 @@ void CaptureDialog::vulkanLayerWarn_mouseClick() { if(info.otherJSONs.size() > 1) msg += - tr("there are other RenderDoc builds registered already. They must be disabled so that " - "capture can happen without nasty clashes."); + tr("there are other conflicting RenderDoc builds registered already. They must be " + "disabled so that vulkan programs can be captured without crashes."); else msg += - tr("there is another RenderDoc build registered already. It must be disabled so that " - "capture can happen without nasty clashes."); + tr("there is another conflicting RenderDoc build registered already. It must be disabled " + "so that vulkan programs can be captured without crashes."); if(!thisRegistered) msg += tr(" Also "); @@ -410,44 +418,69 @@ void CaptureDialog::vulkanLayerWarn_mouseClick() msg += lit("\n"); } + if(needElevation) + msg += + tr("Due to some builds being in privileged locations, RenderDoc must elevate permissions " + "to update them.\n\n"); + msg += tr("This is a one-off change, it won't be needed again unless the installation moves."); QMessageBox::StandardButton install = RDDialog::question(this, caption, msg, RDDialog::YesNoCancel); if(install == QMessageBox::Yes) { - bool run = false; - bool admin = false; + bool admin = needElevation; + bool system = true; // default to system-wide install + bool run = true; // default to running - // if we need to elevate, just try it. - if(needElevation) + // if we could install user-local, ask the user if that's what they want. + if(userRegisterable) { - admin = run = true; - } - // if we could elevate, ask the user - else if(couldElevate) - { - QMessageBox::StandardButton elevate = RDDialog::question( - this, tr("System layer install"), - tr("Do you want to elevate permissions to install the layer at a system level?\n\n" - "If you click 'No', the layer will be installed at a per-user level."), - RDDialog::YesNoCancel); + msg = + tr("Do you want to install the layer at a system level?\n\n" + "If you click 'No', the layer will be installed at a per-user level."); + + if(needElevation) + msg += + tr("\n\nNote that RenderDoc needs to elevate permissions to update the registration " + "regardless."); + else + msg += + tr("\n\nNote that RenderDoc will need to elevate permissions to register at system " + "level."); + + QMessageBox::StandardButton elevate = + RDDialog::question(this, tr("Install at system level"), msg, RDDialog::YesNoCancel); if(elevate == QMessageBox::Yes) - admin = true; + admin = system = true; else if(elevate == QMessageBox::No) - admin = false; + system = false; run = (elevate != QMessageBox::Cancel); } - // otherwise run non-elevated - else - { - run = true; - } if(run) { + auto regComplete = [this, admin]() { + bool needReg = RENDERDOC_NeedVulkanLayerRegistration(NULL); + ui->vulkanLayerWarn->setVisible(needReg); + +#if !defined(Q_OS_LINUX) + // can't alert the user on linux because the command might still be running - there's + // seemingly no portable way to wait for the command to finish. + if(needReg) + { + QString err = tr("Vulkan layer registration failed for unknown reasons."); + + if(admin) + err += tr(" Ensure that the elevation to admin permissions succeeded."); + + RDDialog::critical(this, tr("Layer registration failed"), err); + } +#endif + }; + if(admin) { // linux sometimes can't run GUI apps as root, so we have to run renderdoccmd. Check that it's @@ -480,25 +513,40 @@ void CaptureDialog::vulkanLayerWarn_mouseClick() // it's in the path, we can continue } - RunProcessAsAdmin(cmd, QStringList() << lit("vulkanregister") << lit("--system"), this, - true, [this]() { ui->vulkanLayerWarn->setVisible(false); }); + QStringList renderdoccmdParams; + + renderdoccmdParams << lit("vulkanregister"); + if(system) + renderdoccmdParams << lit("--system"); + + if(!RunProcessAsAdmin(cmd, renderdoccmdParams, this, true, regComplete)) + regComplete(); #else - RunProcessAsAdmin(qApp->applicationFilePath(), - QStringList() << lit("--install_vulkan_layer") << lit("root"), this, - false, [this]() { ui->vulkanLayerWarn->setVisible(false); }); + QStringList qrenderdocParams; + + qrenderdocParams << lit("--install_vulkan_layer"); + if(system) + qrenderdocParams << lit("root"); + + if(!RunProcessAsAdmin(qApp->applicationFilePath(), qrenderdocParams, this, false, regComplete)) + regComplete(); #endif return; } else { QProcess *process = new QProcess; - process->start(qApp->applicationFilePath(), QStringList() << lit("--install_vulkan_layer") - << lit("user")); + process->start(qApp->applicationFilePath(), QStringList() + << lit("--install_vulkan_layer") + << (system ? lit("root") : lit("user"))); process->waitForFinished(300); // when the process exits, delete QObject::connect(process, OverloadedSlot::of(&QProcess::finished), - [process](int exitCode, QProcess::ExitStatus) { process->deleteLater(); }); + [regComplete, process](int exitCode, QProcess::ExitStatus) { + process->deleteLater(); + regComplete(); + }); } } diff --git a/renderdoc/api/replay/replay_enums.h b/renderdoc/api/replay/replay_enums.h index 86d196c6e..900772c9c 100644 --- a/renderdoc/api/replay/replay_enums.h +++ b/renderdoc/api/replay/replay_enums.h @@ -3926,10 +3926,18 @@ DOCUMENT(R"(A set of flags giving details of the current status of vulkan layer Fixing any issues will require elevation to system administrator privileges. -.. data:: CouldElevate +.. data:: UserRegisterable - Fixing issues could be done purely as a user, but can optionally be done at system level with - system administrator privileges. + This layer can be registered as user-local, as well as system-wide. If :data:`NeedElevation` isn't + also set then the entire process can be done un-elevated if user-local is desired. + + .. note:: + + If the :data:`NeedElevation` flag is set then elevation is required to fix the layer + registration, even if a user-local registration is desired. + + Most commonly this situation arises if there is no other registration, or the existing one is + already user-local. .. data:: RegisterAll @@ -3951,7 +3959,7 @@ enum class VulkanLayerFlags : uint32_t OtherInstallsRegistered = 0x1, ThisInstallRegistered = 0x2, NeedElevation = 0x4, - CouldElevate = 0x8, + UserRegisterable = 0x8, RegisterAll = 0x10, UpdateAllowed = 0x20, Unfixable = 0x40, diff --git a/renderdoc/driver/vulkan/vk_posix.cpp b/renderdoc/driver/vulkan/vk_posix.cpp index e4e6bd068..18f580373 100644 --- a/renderdoc/driver/vulkan/vk_posix.cpp +++ b/renderdoc/driver/vulkan/vk_posix.cpp @@ -457,7 +457,7 @@ bool VulkanReplay::CheckVulkanLayer(VulkanLayerFlags &flags, std::vector= 1) flags |= VulkanLayerFlags::ThisInstallRegistered; @@ -466,11 +466,23 @@ bool VulkanReplay::CheckVulkanLayer(VulkanLayerFlags &flags, std::vector