From 42841e23bec8866527a16d8e59514db54bbad08f Mon Sep 17 00:00:00 2001 From: baldurk Date: Mon, 20 Jan 2020 11:46:48 +0000 Subject: [PATCH] Expose vulkan layer registration in renderdoccmd on win32. Closes #1690 * This is needed so that the functional tests can elevate and run renderdoccmd to register the vulkan layer, if needed. * At the same time remove the old spammy message and ignore flag - this dates back to before the UI existed, and that should be the way users run RenderDoc generally and it has a good UI for walking through layer registration if needed. * The command is always available, but will only show up in help if attention is needed. * Also fix registering installs on shared drives. --- qrenderdoc/Windows/Dialogs/CaptureDialog.cpp | 4 +- renderdoc/api/replay/replay_enums.h | 5 + renderdoc/core/core.h | 2 + renderdoc/driver/vulkan/vk_posix.cpp | 28 +- renderdoccmd/renderdoccmd.cpp | 315 +++++++++++++++---- renderdoccmd/renderdoccmd_linux.cpp | 151 --------- util/test/rdtest/runner.py | 8 + 7 files changed, 292 insertions(+), 221 deletions(-) diff --git a/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp b/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp index 567a3f7d5..9e405949e 100644 --- a/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp +++ b/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp @@ -515,9 +515,11 @@ void CaptureDialog::vulkanLayerWarn_mouseClick() QStringList renderdoccmdParams; - renderdoccmdParams << lit("vulkanregister"); + renderdoccmdParams << lit("vulkanlayer"); if(system) renderdoccmdParams << lit("--system"); + else + renderdoccmdParams << lit("--user"); if(!RunProcessAsAdmin(cmd, renderdoccmdParams, this, true, regComplete)) regComplete(); diff --git a/renderdoc/api/replay/replay_enums.h b/renderdoc/api/replay/replay_enums.h index e776d3346..d7839626c 100644 --- a/renderdoc/api/replay/replay_enums.h +++ b/renderdoc/api/replay/replay_enums.h @@ -4126,6 +4126,10 @@ DOCUMENT(R"(A set of flags giving details of the current status of vulkan layer .. data:: Unfixable The current situation is not fixable automatically and requires user intervention/disambiguation. + +.. data:: Unsupported + + Vulkan is not supported by this build of RenderDoc and the layer cannot be registered. )"); enum class VulkanLayerFlags : uint32_t { @@ -4137,6 +4141,7 @@ enum class VulkanLayerFlags : uint32_t RegisterAll = 0x10, UpdateAllowed = 0x20, Unfixable = 0x40, + Unsupported = 0x80, }; BITMASK_OPERATORS(VulkanLayerFlags); diff --git a/renderdoc/core/core.h b/renderdoc/core/core.h index 4440714df..f18fae13b 100644 --- a/renderdoc/core/core.h +++ b/renderdoc/core/core.h @@ -475,6 +475,8 @@ public: if(m_VulkanCheck) return m_VulkanCheck(flags, myJSONs, otherJSONs); + flags = VulkanLayerFlags::Unfixable | VulkanLayerFlags::Unsupported; + return false; } diff --git a/renderdoc/driver/vulkan/vk_posix.cpp b/renderdoc/driver/vulkan/vk_posix.cpp index efc2511a8..0d6042634 100644 --- a/renderdoc/driver/vulkan/vk_posix.cpp +++ b/renderdoc/driver/vulkan/vk_posix.cpp @@ -29,6 +29,8 @@ #include "vk_replay.h" #include +#include +#include #include #include @@ -337,6 +339,14 @@ static rdcstr GetSOFromJSON(const rdcstr &json) delete[] json_string; + // get the realpath, if this is a real filename + char *resolved = realpath(ret.c_str(), NULL); + if(resolved && resolved[0]) + { + ret = resolved; + free(resolved); + } + return ret; } @@ -391,17 +401,6 @@ void MakeParentDirs(rdcstr file) bool VulkanReplay::CheckVulkanLayer(VulkanLayerFlags &flags, rdcarray &myJSONs, rdcarray &otherJSONs) { - // see if the user has suppressed all this checking as a "I know what I'm doing" measure - - const char *home_path = getenv("HOME"); - if(home_path == NULL) - home_path = ""; - if(FileExists(rdcstr(home_path) + "/.renderdoc/ignore_vulkan_layer_issues")) - { - flags = VulkanLayerFlags::ThisInstallRegistered; - return false; - } - //////////////////////////////////////////////////////////////////////////////////////// // check that there's only one layer registered, and it points to the same .so file that // we are running with in this instance of renderdoccmd @@ -409,6 +408,13 @@ bool VulkanReplay::CheckVulkanLayer(VulkanLayerFlags &flags, rdcarray &m rdcstr librenderdoc_path; FileIO::GetLibraryFilename(librenderdoc_path); + char *resolved = realpath(librenderdoc_path.c_str(), NULL); + if(resolved && resolved[0]) + { + librenderdoc_path = resolved; + free(resolved); + } + if(librenderdoc_path.empty() || !FileExists(librenderdoc_path)) { RDCERR("Couldn't determine current library path!"); diff --git a/renderdoccmd/renderdoccmd.cpp b/renderdoccmd/renderdoccmd.cpp index 5b9caeee2..564945857 100644 --- a/renderdoccmd/renderdoccmd.cpp +++ b/renderdoccmd/renderdoccmd.cpp @@ -38,6 +38,8 @@ std::string conv(const rdcstr &s) return std::string(s.begin(), s.end()); } +static int command_usage(std::string command = ""); + // normally this is in the renderdoc core library, but it's needed for the 'unknown enum' path, // so we implement it here using ostringstream. It's not great, but this is a very uncommon path - // either for invalid values or for when a new enum is added and the code isn't updated @@ -122,64 +124,6 @@ void DisplayRendererPreview(IReplayController *renderer, uint32_t width, uint32_ DisplayRendererPreview(renderer, d, width, height, numLoops); } -std::map commands; -std::map aliases; - -void add_command(const std::string &name, Command *cmd) -{ - commands[name] = cmd; -} - -void add_alias(const std::string &alias, const std::string &command) -{ - aliases[alias] = command; -} - -static void clean_up() -{ - for(auto it = commands.begin(); it != commands.end(); ++it) - delete it->second; -} - -static int command_usage(std::string command = "") -{ - if(!command.empty()) - std::cerr << command << " is not a valid command." << std::endl << std::endl; - - std::cerr << "Usage: renderdoccmd [args ...]" << std::endl; - std::cerr << "Command line tool for capture & replay with RenderDoc." << std::endl << std::endl; - - std::cerr << "Command can be one of:" << std::endl; - - size_t max_width = 0; - for(auto it = commands.begin(); it != commands.end(); ++it) - { - if(it->second->IsInternalOnly()) - continue; - - max_width = std::max(max_width, it->first.length()); - } - - for(auto it = commands.begin(); it != commands.end(); ++it) - { - if(it->second->IsInternalOnly()) - continue; - - std::cerr << " " << it->first; - for(size_t n = it->first.length(); n < max_width + 4; n++) - std::cerr << ' '; - std::cerr << it->second->Description() << std::endl; - } - std::cerr << std::endl; - - std::cerr << "To see details of any command, see 'renderdoccmd --help'" << std::endl - << std::endl; - - std::cerr << "For more information, see ." << std::endl; - - return 2; -} - static std::vector version_lines; struct VersionCommand : public Command @@ -1184,13 +1128,268 @@ struct EmbeddedSectionCommand : public Command } }; +struct VulkanRegisterCommand : public Command +{ +private: + bool m_LayerNeedUpdate = false; + VulkanLayerRegistrationInfo m_Info; + +public: + VulkanRegisterCommand(const GlobalEnvironment &env) : Command(env) + { + m_LayerNeedUpdate = RENDERDOC_NeedVulkanLayerRegistration(&m_Info); + } + virtual void AddOptions(cmdline::parser &parser) + { + parser.add("explain", '\0', + "Explain what the status of the layer registration is, and how it can be resolved"); + parser.add("register", '\0', "Register RenderDoc's vulkan layer"); + parser.add("user", '\0', + "Install layer registration at user-local level instead of system-wide"); + parser.add("system", '\0', + "Install layer registration system-wide (requires admin privileges)"); + } + virtual const char *Description() { return "Vulkan layer registration needs attention"; } + virtual bool IsInternalOnly() + { + // if the layer is registered and doesn't need an update, don't report this command in help + return !m_LayerNeedUpdate; + } + virtual bool IsCaptureCommand() { return false; } + virtual int Execute(cmdline::parser &parser, const CaptureOptions &) + { + if(parser.exist("explain") || !parser.exist("register")) + { + if(m_LayerNeedUpdate) + { + if(m_Info.flags & VulkanLayerFlags::Unfixable) + { + std::cerr << "** There is an unfixable problem with your vulkan layer configuration.\n\n" + "This is most commonly caused by having a distribution-provided package of " + "RenderDoc " + "installed, which cannot be modified by another build of RenderDoc.\n\n" + "Please consult the RenderDoc documentation, or package/distribution " + "documentation on " + "linux." + << std::endl; + + if(m_Info.otherJSONs.size() > 1) + std::cerr << "Conflicting manifests:\n\n"; + else + std::cerr << "Conflicting manifest:\n\n"; + + for(const rdcstr &j : m_Info.otherJSONs) + std::cerr << conv(j) << std::endl; + + return 0; + } + + std::cerr << "*************************************************************************" + << std::endl; + std::cerr << "** Warning: Vulkan layer not correctly registered. **" + << std::endl; + std::cerr << std::endl; + + if(m_Info.flags & VulkanLayerFlags::OtherInstallsRegistered) + std::cerr << " - Non-matching RenderDoc layer(s) are registered." << std::endl; + + if(!(m_Info.flags & VulkanLayerFlags::ThisInstallRegistered)) + std::cerr << " - This build's RenderDoc layer is not registered." << std::endl; + + std::cerr << std::endl; + + std::cerr << " To fix this, the following actions must take place: " << std::endl + << std::endl; + + const bool registerAll = bool(m_Info.flags & VulkanLayerFlags::RegisterAll); + const bool updateAllowed = bool(m_Info.flags & VulkanLayerFlags::UpdateAllowed); + + for(const rdcstr &j : m_Info.otherJSONs) + std::cerr << (updateAllowed ? " Unregister/update: " : " Unregister: ") << j.c_str() + << std::endl; + + if(!(m_Info.flags & VulkanLayerFlags::ThisInstallRegistered)) + { + if(registerAll) + { + for(const rdcstr &j : m_Info.myJSONs) + std::cerr << (updateAllowed ? " Register/update: " : " Register: ") << j.c_str() + << std::endl; + } + else + { + std::cerr << (updateAllowed ? " Register one of:" : " Register/update one of:") + << std::endl; + for(const rdcstr &j : m_Info.myJSONs) + std::cerr << " -- " << j.c_str() << "\n"; + } + } + + std::cerr << std::endl; + + if(m_Info.flags & VulkanLayerFlags::UserRegisterable) + { + std::cerr << " You must choose whether to register at user or system level." << std::endl + << std::endl; + std::cerr + << " 'vulkanlayer --register --user' will register the layer local to your user." + << std::endl; + if(m_Info.flags & VulkanLayerFlags::NeedElevation) + std::cerr << " (This requires admin permissions to unregister other installs)" + << std::endl; + else + std::cerr << " (This does not require admin permission)" << std::endl; + std::cerr << std::endl; + std::cerr << " If you want to install system-wide, run 'vulkanlayer --system'." + << std::endl; + std::cerr << " (This requires admin permission)" << std::endl; + + std::cerr << "*************************************************************************" + << std::endl; + std::cerr << std::endl; + } + else + { + std::cerr << " The layer must be registered at system level, this operation requires\n" + << " admin permissions." << std::endl; + std::cerr << std::endl; + std::cerr << " Run 'vulkanlayer --register --system' as administrator to register." + << std::endl; + + std::cerr << std::endl; + std::cerr << "*************************************************************************" + << std::endl; + std::cerr << std::endl; + } + } + else + { + std::cerr << "The RenderDoc vulkan layer appears to be correctly registered." << std::endl; + } + + // don't do anything if we're just explaining the situation + return 0; + } + + bool user = parser.exist("user"), system = parser.exist("system"); + + if(m_Info.flags & VulkanLayerFlags::UserRegisterable) + { + if(user) + { + std::cerr << "Vulkan layer cannot be registered at user level." << std::endl; + return 1; + } + } + + if(user && system) + { + std::cerr << "Vulkan layer cannot be registered at user and system levels." << std::endl; + return 1; + } + else if(user || system) + { + RENDERDOC_UpdateVulkanLayerRegistration(system); + + if(RENDERDOC_NeedVulkanLayerRegistration(NULL)) + { + std::cerr << "Vulkan layer registration not successful. "; + if(system) + std::cerr << "Check that you are running as administrator"; + std::cerr << std::endl; + } + } + else + { + std::cerr << "You must select either '--user' or '--system' to choose where to register the " + "vulkan layer." + << std::endl; + return 1; + } + + return 0; + } +}; + REPLAY_PROGRAM_MARKER() +VulkanRegisterCommand *vulkan = NULL; + +std::map commands; +std::map aliases; + +void add_command(const std::string &name, Command *cmd) +{ + commands[name] = cmd; +} + +void add_alias(const std::string &alias, const std::string &command) +{ + aliases[alias] = command; +} + +static void clean_up() +{ + for(auto it = commands.begin(); it != commands.end(); ++it) + delete it->second; +} + +static int command_usage(std::string command) +{ + if(!command.empty()) + std::cerr << command << " is not a valid command." << std::endl << std::endl; + + if(vulkan && !vulkan->IsInternalOnly()) + std::cerr << "** NOTE: Vulkan layer registration problem detected.\n" + "** Run 'vulkanlayer --explain' for more details" + << std::endl + << std::endl; + + std::cerr << "Usage: renderdoccmd [args ...]" << std::endl; + std::cerr << "Command line tool for capture & replay with RenderDoc." << std::endl << std::endl; + + std::cerr << "Command can be one of:" << std::endl; + + size_t max_width = 0; + for(auto it = commands.begin(); it != commands.end(); ++it) + { + if(it->second->IsInternalOnly()) + continue; + + max_width = std::max(max_width, it->first.length()); + } + + for(auto it = commands.begin(); it != commands.end(); ++it) + { + if(it->second->IsInternalOnly()) + continue; + + std::cerr << " " << it->first; + for(size_t n = it->first.length(); n < max_width + 4; n++) + std::cerr << ' '; + std::cerr << it->second->Description() << std::endl; + } + std::cerr << std::endl; + + std::cerr << "To see details of any command, see 'renderdoccmd --help'" << std::endl + << std::endl; + + std::cerr << "For more information, see ." << std::endl; + + return 2; +} + int renderdoccmd(GlobalEnvironment &env, std::vector &argv) { // we don't need this in renderdoccmd. env.enumerateGPUs = false; + vulkan = new VulkanRegisterCommand(env); + + // if vulkan isn't supported, or the layer is fully registered, this command will not be listed + // in help so it will be invisible + add_command("vulkanlayer", vulkan); + try { // add basic commands, and common aliases diff --git a/renderdoccmd/renderdoccmd_linux.cpp b/renderdoccmd/renderdoccmd_linux.cpp index 8b6e9af15..eb9822633 100644 --- a/renderdoccmd/renderdoccmd_linux.cpp +++ b/renderdoccmd/renderdoccmd_linux.cpp @@ -48,153 +48,6 @@ void Daemonise() daemon(1, 0); } -struct VulkanRegisterCommand : public Command -{ - VulkanRegisterCommand(const GlobalEnvironment &env) : Command(env) {} - virtual void AddOptions(cmdline::parser &parser) - { - parser.add("ignore", 'i', "Do nothing and don't warn about Vulkan layer issues."); - parser.add( - "system", '\0', - "Install layer registration to /etc instead of $HOME/.local (requires root privileges)"); - } - virtual const char *Description() - { - return "Attempt to automatically fix Vulkan layer registration issues"; - } - virtual bool IsInternalOnly() { return false; } - virtual bool IsCaptureCommand() { return false; } - virtual int Execute(cmdline::parser &parser, const CaptureOptions &) - { - bool ignore = (parser.exist("ignore")); - - if(ignore) - { - std::cout << "Not fixing vulkan layer issues, and suppressing future warnings." << std::endl; - std::cout << "To undo, remove '$HOME/.renderdoc/ignore_vulkan_layer_issues'." << std::endl; - - std::string ignorePath = std::string(getenv("HOME")) + "/.renderdoc/"; - - mkdir(ignorePath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); - - ignorePath += "ignore_vulkan_layer"; - - FILE *f = fopen(ignorePath.c_str(), "w"); - - if(f) - { - fputs("This file suppresses any checks for vulkan layer issues.\n", f); - fputs("Delete this file to restore default checking.\n", f); - - fclose(f); - } - else - { - std::cerr << "Couldn't create '$HOME/.renderdoc/ignore_vulkan_layer_issues'." << std::endl; - } - - return 0; - } - - RENDERDOC_UpdateVulkanLayerRegistration(parser.exist("system")); - - return 0; - } -}; - -void VerifyVulkanLayer(const GlobalEnvironment &env, int argc, char *argv[]) -{ - VulkanLayerRegistrationInfo info; - - bool needUpdate = RENDERDOC_NeedVulkanLayerRegistration(&info); - - if(!needUpdate) - { - if(!(info.flags & VulkanLayerFlags::Unfixable)) - add_command("vulkanregister", new VulkanRegisterCommand(env)); - return; - } - - std::cerr << "*************************************************************************" - << std::endl; - std::cerr << "** Warning: Vulkan capture possibly not configured. **" - << std::endl; - std::cerr << std::endl; - - if(info.flags & VulkanLayerFlags::OtherInstallsRegistered) - std::cerr << "Multiple RenderDoc layers are registered, possibly from different builds." - << std::endl; - - if(!(info.flags & VulkanLayerFlags::ThisInstallRegistered)) - std::cerr << "This build's RenderDoc layer is not registered." << std::endl; - - std::cerr << "To fix this, the following actions must take place: " << std::endl << std::endl; - - const bool registerAll = bool(info.flags & VulkanLayerFlags::RegisterAll); - const bool updateAllowed = bool(info.flags & VulkanLayerFlags::UpdateAllowed); - - for(const rdcstr &j : info.otherJSONs) - std::cerr << (updateAllowed ? "Unregister/update: " : "Unregister: ") << j.c_str() << std::endl; - - if(!(info.flags & VulkanLayerFlags::ThisInstallRegistered)) - { - if(registerAll) - { - for(const rdcstr &j : info.myJSONs) - std::cerr << (updateAllowed ? "Register/update: " : "Register: ") << j.c_str() << std::endl; - } - else - { - std::cerr << (updateAllowed ? "Register one of:" : "Register/update one of:") << std::endl; - for(const rdcstr &j : info.myJSONs) - std::cerr << " -- " << j.c_str() << "\n"; - } - } - - std::cerr << std::endl; - - if(info.flags & VulkanLayerFlags::Unfixable) - { - std::cerr << "NOTE: The renderdoc layer registered in /usr is reserved for distribution" - << std::endl; - std::cerr << "controlled packages. RenderDoc cannot automatically unregister this even" - << std::endl; - std::cerr << "with root permissions, you must fix this conflict manually." << std::endl - << std::endl; - - std::cerr << "*************************************************************************" - << std::endl; - std::cerr << std::endl; - - return; - } - - std::cerr << "NOTE: Automatically removing or changing the layer registered in /etc" << std::endl; - std::cerr << "will require root privileges." << std::endl << std::endl; - - std::cerr << "To fix these issues run the 'vulkanregister' command." << std::endl; - std::cerr << "Use 'vulkanregister --help' to see more information." << std::endl; - std::cerr << std::endl; - - std::cerr << "By default 'vulkanregister' will register the layer to your $HOME folder." - << std::endl; - std::cerr << "This does not require root permissions." << std::endl; - std::cerr << std::endl; - std::cerr << "If you want to install to the system, run 'vulkanregister --system'." << std::endl; - std::cerr << "This requires root permissions to write to /etc/vulkan/." << std::endl; - - // just in case there's a strange install that is misdetected or something then allow - // users to suppress this message and just say "I know what I'm doing". - std::cerr << std::endl; - std::cerr << "To suppress this warning in future, run 'vulkanregister --ignore'." << std::endl; - - std::cerr << "*************************************************************************" - << std::endl; - std::cerr << std::endl; - - add_command("vulkanregister", new VulkanRegisterCommand(env)); -} - static Display *display = NULL; WindowingData DisplayRemoteServerPreview(bool active, const rdcarray &systems) @@ -486,10 +339,6 @@ int main(int argc, char *argv[]) display = env.xlibDisplay = XOpenDisplay(NULL); #endif -#if defined(RENDERDOC_SUPPORT_VULKAN) - VerifyVulkanLayer(env, argc, argv); -#endif - // add compiled-in support to version line { std::string support = "APIs supported at compile-time: "; diff --git a/util/test/rdtest/runner.py b/util/test/rdtest/runner.py index d7eace3a9..89cf5fc57 100644 --- a/util/test/rdtest/runner.py +++ b/util/test/rdtest/runner.py @@ -12,6 +12,7 @@ import renderdoc as rd from . import util from . import testcase from .logging import log +from pathlib import Path def get_tests(): @@ -225,6 +226,13 @@ def run_tests(test_include: str, test_exclude: str, in_process: bool, slow_tests args = sys.argv.copy() args.append("--internal_vulkan_register") + for i in range(len(args)): + if os.path.exists(args[i]): + args[i] = str(Path(args[i]).resolve()) + + if 'renderdoccmd' in sys.executable: + args = ['vulkanlayer', '--register', '--system'] + ctypes.windll.shell32.ShellExecuteW(None, "runas", sys.executable, ' '.join(args), None, 1) time.sleep(10)