From d3e4790ee834ec949e7269b95d38d33249e149bb Mon Sep 17 00:00:00 2001 From: baldurk Date: Fri, 26 Aug 2016 13:35:31 +0200 Subject: [PATCH] Add command to renderdoccmd that will handle vulkan layer registration * This is similar to the button on the capture dialog of the windows UI, except on linux the situation is a bit more complex as layers can be registered in two different system locations (one for distro packages and one for non-distro stuff), as well as in $HOME. * On linux we make sure the json always contains an absolute path so we can tell from any given build whether or not a json is our own or not. * By default we now install the json to /etc since /usr is reserved for proper distro packages, and shouldn't be written to by 'make install'. There's a CMake variable for overriding this for anyone who wants to make install into a custom folder (maybe for packaging). --- renderdoc/CMakeLists.txt | 3 +- renderdoc/driver/vulkan/CMakeLists.txt | 10 +- renderdoccmd/renderdoccmd_linux.cpp | 477 +++++++++++++++++++++++++ 3 files changed, 487 insertions(+), 3 deletions(-) diff --git a/renderdoc/CMakeLists.txt b/renderdoc/CMakeLists.txt index afa9537fa..f23495a91 100644 --- a/renderdoc/CMakeLists.txt +++ b/renderdoc/CMakeLists.txt @@ -228,7 +228,8 @@ set(data data/glsl/text.vert data/glsl/array2ms.comp data/glsl/ms2array.comp - data/sourcecodepro.ttf) + data/sourcecodepro.ttf + driver/vulkan/renderdoc.json) set(data_objects) diff --git a/renderdoc/driver/vulkan/CMakeLists.txt b/renderdoc/driver/vulkan/CMakeLists.txt index 465575442..1cb680fef 100644 --- a/renderdoc/driver/vulkan/CMakeLists.txt +++ b/renderdoc/driver/vulkan/CMakeLists.txt @@ -40,6 +40,12 @@ set(sources set(definitions ${RDOC_DEFINITIONS}) +# This must be specified separately because it needs to go in /etc regardless of what the install +# prefix is, since the loader only looks in a set location (and /usr/share is reserved for distribution +# packages). For people who want to 'make install' to another folder, perhaps for preparing a package, +# they can set this variable to make sure it stays local +set(VULKAN_LAYER_FOLDER /etc/vulkan/implicit_layer.d CACHE PATH "Path to install the vulkan layer file") + if(ANDROID) list(APPEND sources vk_posix.cpp vk_android.cpp vk_tracelayer_android.cpp) list(APPEND definitions PRIVATE -DVK_USE_PLATFORM_ANDROID_KHR) @@ -57,11 +63,11 @@ elseif(UNIX) set(json_in ${CMAKE_CURRENT_SOURCE_DIR}/renderdoc.json) set(json_out ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/renderdoc_capture.json) add_custom_command(OUTPUT ${json_out} - COMMAND sed '{s%...renderdoc.dll%librenderdoc.so%}' < ${json_in} > ${json_out} + COMMAND sed '{s%...renderdoc.dll%${CMAKE_INSTALL_PREFIX}/lib/librenderdoc.so%}' < ${json_in} > ${json_out} DEPENDS ${json_in}) add_custom_target(generate-json ALL DEPENDS ${json_out}) - install (FILES ${json_out} DESTINATION share/vulkan/implicit_layer.d) + install (FILES ${json_out} DESTINATION ${VULKAN_LAYER_FOLDER}) endif() # GCC 6.1 may or may not complain about enum reference casts diff --git a/renderdoccmd/renderdoccmd_linux.cpp b/renderdoccmd/renderdoccmd_linux.cpp index b1ea114d6..4ec264ac6 100644 --- a/renderdoccmd/renderdoccmd_linux.cpp +++ b/renderdoccmd/renderdoccmd_linux.cpp @@ -24,10 +24,15 @@ ******************************************************************************/ #include "renderdoccmd.h" +#include #include +#include #include #include +#include #include +#include +#include #include #include @@ -38,6 +43,7 @@ #include using std::string; +using std::vector; void Daemonise() { @@ -45,6 +51,473 @@ void Daemonise() daemon(1, 0); } +// this is included as a data file in librenderdoc.so +extern unsigned char driver_vulkan_renderdoc_json[]; +extern int driver_vulkan_renderdoc_json_len; + +static string GenerateJSON(const string &sopath) +{ + char *txt = (char *)driver_vulkan_renderdoc_json; + int len = driver_vulkan_renderdoc_json_len; + + string json = string(txt, txt + len); + + const char dllPathString[] = ".\\\\renderdoc.dll"; + + size_t idx = json.find(dllPathString); + + return json.substr(0, idx) + sopath + json.substr(idx + sizeof(dllPathString) - 1); +} + +static bool FileExists(const string &path) +{ + FILE *f = fopen(path.c_str(), "r"); + + if(f) + { + fclose(f); + return true; + } + + return false; +} + +static string GetSOFromJSON(const string &json) +{ + char *json_string = new char[1024]; + memset(json_string, 0, 1024); + + FILE *f = fopen(json.c_str(), "r"); + + if(f) + { + fread(json_string, 1, 1024, f); + + fclose(f); + } + + string ret = ""; + + // The line is: + // "library_path": "/foo/bar/librenderdoc.so", + char *c = strstr(json_string, "library_path"); + + if(c) + { + c += sizeof("library_path\": \"") - 1; + + char *quote = strchr(c, '"'); + + if(quote) + { + *quote = 0; + ret = c; + } + } + + delete[] json_string; + + return ret; +} + +enum +{ + USR, + ETC, + HOME, + COUNT +}; + +string layerRegistrationPath[COUNT] = { + "/usr/share/vulkan/implicit_layer.d/renderdoc_capture.json", + "/etc/vulkan/implicit_layer.d/renderdoc_capture.json", + string(getenv("HOME")) + "/.local/share/vulkan/implicit_layer.d/renderdoc_capture.json"}; + +struct VulkanRegisterCommand : public Command +{ + VulkanRegisterCommand(bool layer_exists[COUNT], const string &path) + { + etcExists = layer_exists[ETC]; + homeExists = layer_exists[HOME]; + libPath = path; + } + + virtual void AddOptions(cmdline::parser &parser) + { + parser.add("ignore", 'i', "Do nothing and don't warn about Vulkan layer issues."); + parser.add( + "system", '\0', + "Install layer registration to /etc instead of $HOME/.local (requires root privileges)"); + parser.add("dry-run", 'n', "Don't perform any actions, instead print what would happen."); + } + virtual const char *Description() + { + return "Attempt to automatically fix Vulkan layer registration issues"; + } + virtual bool IsInternalOnly() { return false; } + virtual bool IsCaptureCommand() { return false; } + virtual int Execute(cmdline::parser &parser, const CaptureOptions &) + { + bool ignore = (parser.exist("ignore")); + + if(ignore) + { + std::cout << "Not fixing vulkan layer issues, and suppressing future warnings." << std::endl; + std::cout << "To undo, remove '$HOME/.renderdoc/ignore_vulkan_layer_issues'." << std::endl; + + string ignorePath = string(getenv("HOME")) + "/.renderdoc/"; + + mkdir(ignorePath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); + + ignorePath += "ignore_vulkan_layer"; + + FILE *f = fopen(ignorePath.c_str(), "w"); + + if(f) + { + fputs("This file suppresses any checks for vulkan layer issues.\n", f); + fputs("Delete this file to restore default checking.\n", f); + + fclose(f); + } + else + { + std::cerr << "Couldn't create '$HOME/.renderdoc/ignore_vulkan_layer_issues'." << std::endl; + } + + return 0; + } + + bool system = (parser.exist("system")); + bool dryrun = (parser.exist("dry-run")); + + // if we want to install to the system and there's a registration in $HOME, delete it + if(system && homeExists) + { + std::cout << "Removing '" << layerRegistrationPath[HOME] << "'" << std::endl; + + if(!dryrun) + { + int ret = unlink(layerRegistrationPath[HOME].c_str()); + + if(ret < 0) + { + const char *const errtext = strerror(errno); + std::cout << "Error - " << errtext << std::endl; + } + } + } + + // and vice-versa + if(!system && etcExists) + { + std::cout << "Removing '" << layerRegistrationPath[ETC] << "'" << std::endl; + + if(!dryrun) + { + int ret = unlink(layerRegistrationPath[ETC].c_str()); + + if(ret < 0) + { + const char *const errtext = strerror(errno); + std::cout << "Error - " << errtext << std::endl; + } + } + } + + int idx = system ? ETC : HOME; + + string path = GetSOFromJSON(layerRegistrationPath[idx]); + + if(path != libPath) + { + if((system && !etcExists) || (!system && !homeExists)) + { + std::cout << "Registering '" << layerRegistrationPath[idx] << "'" << std::endl; + } + else + { + std::cout << "Updating '" << layerRegistrationPath[idx] << "'" << std::endl; + if(path == "") + { + std::cout << " JSON is corrupt or unrecognised, replacing with valid JSON pointing" + << std::endl; + std::cout << " to '" << libPath << "'" << std::endl; + } + else + { + std::cout << " Repointing from '" << path << "'" << std::endl; + std::cout << " to '" << libPath << "'" << std::endl; + } + } + + if(!dryrun) + { + FILE *f = fopen(layerRegistrationPath[idx].c_str(), "w"); + + if(f) + { + fputs(GenerateJSON(libPath).c_str(), f); + + fclose(f); + } + else + { + const char *const errtext = strerror(errno); + std::cout << "Error - " << errtext << std::endl; + } + } + } + + return 0; + } + + bool etcExists; + bool homeExists; + string libPath; +}; + +static void VerifyVulkanLayer(int argc, char *argv[]) +{ + // see if the user has suppressed all this checking as a "I know what I'm doing" measure + + string ignorePath = string(getenv("HOME")) + "/.renderdoc/ignore_vulkan_layer_issues"; + if(FileExists(ignorePath)) + return; + + //////////////////////////////////////////////////////////////////////////////////////// + // check that there's only one layer registered, and it points to the same .so file that + // we are running with in this instance of renderdoccmd + + // this is a hack, but the only reliable way to find the absolute path to the library. + // dladdr would be fine but it returns the wrong result for symbols in the library + string librenderdoc_path; + + { + FILE *f = fopen("/proc/self/maps", "r"); + + if(f) + { + // read the whole thing in one go. There's no need to try and be tight with + // this allocation, so just make sure we can read everything. + char *map_string = new char[1024 * 1024]; + memset(map_string, 0, 1024 * 1024); + + fread(map_string, 1, 1024 * 1024, f); + + fclose(f); + + char *c = strstr(map_string, "/librenderdoc.so"); + + if(c) + { + // walk backwards until we hit the start of the line + while(c > map_string) + { + c--; + + if(c[0] == '\n') + { + c++; + break; + } + } + + // walk forwards across the address range (00400000-0040c000) + while(isalnum(c[0]) || c[0] == '-') + c++; + + // whitespace + while(c[0] == ' ') + c++; + + // permissions (r-xp) + while(isalpha(c[0]) || c[0] == '-') + c++; + + // whitespace + while(c[0] == ' ') + c++; + + // offset (0000b000) + while(isalnum(c[0]) || c[0] == '-') + c++; + + // whitespace + while(c[0] == ' ') + c++; + + // dev + while(isdigit(c[0]) || c[0] == ':') + c++; + + // whitespace + while(c[0] == ' ') + c++; + + // inode + while(isdigit(c[0])) + c++; + + // whitespace + while(c[0] == ' ') + c++; + + // FINALLY we are at the start of the actual path + char *end = strchr(c, '\n'); + + if(end) + { + librenderdoc_path = string(c, end - c); + } + } + + delete[] map_string; + } + } + + // it's impractical to determine whether the currently running RenderDoc build is just a loose + // extract of a tarball or a distribution that decided to put all the files in the same folder, + // and whether or not the library is in ld's searchpath. + // + // Instead we just make the requirement that renderdoc.json will always contain an absolute path + // to the matching librenderdoc.so, so that we can check if it points to this build or another + // build etc. + // + // Note there are three places to register layers - /usr, /etc and /home. The first is reserved + // for distribution packages, so if it conflicts or needs to be deleted for this install to run, + // we can't do that and have to just prompt the user. /etc we can mess with since that's for + // non-distribution packages, but it will need root permissions. + + bool exist[COUNT]; + bool match[COUNT]; + + int numExist = 0; + int numMatch = 0; + + for(int i = 0; i < COUNT; i++) + { + exist[i] = FileExists(layerRegistrationPath[i]); + match[i] = (GetSOFromJSON(layerRegistrationPath[i]) == librenderdoc_path); + + if(exist[i]) + numExist++; + + if(match[i]) + numMatch++; + } + + // if we only have one registration, check that it points to us. If so, we're good + if(numExist == 1 && numMatch == 1) + return; + + // if we're about to execute the command, don't print all this explanatory text. + if(argc > 1 && !strcmp(argv[1], "vulkanregister")) + { + add_command("vulkanregister", new VulkanRegisterCommand(exist, librenderdoc_path)); + return; + } + + std::cerr << "*************************************************************************" + << std::endl; + std::cerr << "** Warning: Vulkan capture possibly not configured. **" + << std::endl; + std::cerr << std::endl; + if(numExist > 1) + std::cerr << "Multiple RenderDoc layers are registered, possibly from different builds." + << std::endl; + else if(numExist < 0) + std::cerr << "RenderDoc layer is not registered." << std::endl; + else + std::cerr << "RenderDoc layer is registered, but to a different library." << std::endl; + std::cerr << "To fix this, the following actions must take place: " << std::endl << std::endl; + + bool printed = false; + + if(exist[USR] && !match[USR]) + { + std::cerr << "* Unregister: '" << layerRegistrationPath[USR] << "'" << std::endl; + printed = true; + } + + if(exist[ETC] && !match[ETC]) + { + std::cerr << "* Unregister or update: '" << layerRegistrationPath[ETC] << "'" << std::endl; + printed = true; + } + + if(exist[HOME] && !match[HOME]) + { + std::cerr << "* Unregister or update: '" << layerRegistrationPath[HOME] << "'" << std::endl; + printed = true; + } + + if(printed) + std::cerr << std::endl; + + if(exist[USR] && match[USR]) + { + // just need to unregister others + } + else + { + if(!exist[ETC] && !exist[HOME]) + { + std::cerr << "* Register either: '" << layerRegistrationPath[ETC] << "'" << std::endl; + std::cerr << " or: '" << layerRegistrationPath[HOME] << "'" << std::endl; + } + else + { + std::cerr << "* Update or register either: '" << layerRegistrationPath[ETC] << "'" << std::endl; + std::cerr << " or: '" << layerRegistrationPath[HOME] << "'" + << std::endl; + } + + std::cerr << std::endl; + } + + if(exist[USR] && !match[USR]) + { + std::cerr << "NOTE: The renderdoc layer registered in /usr is reserved for distribution" + << std::endl; + std::cerr << "controlled packages. RenderDoc cannot automatically unregister this even" + << std::endl; + std::cerr << "with root permissions, you must fix this conflict manually." << std::endl + << std::endl; + + std::cerr << "*************************************************************************" + << std::endl; + std::cerr << std::endl; + + return; + } + + std::cerr << "NOTE: Automatically removing or changing the layer registered in /etc" << std::endl; + std::cerr << "will require root privileges." << std::endl << std::endl; + + std::cerr << "To fix these issues run the 'vulkanregister' command." << std::endl; + std::cerr << "Use 'vulkanregister --help' to see more information." << std::endl; + std::cerr << std::endl; + + std::cerr << "By default 'vulkanregister' will register the layer to your $HOME folder." + << std::endl; + std::cerr << "This does not require root permissions." << std::endl; + std::cerr << std::endl; + std::cerr << "If you want to install to the system, run 'vulkanregister --system'." << std::endl; + std::cerr << "This requires root permissions to write to /etc/vulkan/." << std::endl; + + // just in case there's a strange install that is misdetected or something then allow + // users to suppress this message and just say "I know what I'm doing". + std::cerr << std::endl; + std::cerr << "To suppress this warning in future, run 'vulkanregister --ignore'." << std::endl; + + std::cerr << "*************************************************************************" + << std::endl; + std::cerr << std::endl; + + add_command("vulkanregister", new VulkanRegisterCommand(exist, librenderdoc_path)); +} + void DisplayRendererPreview(ReplayRenderer *renderer, TextureDisplay &displayCfg, uint32_t width, uint32_t height) { @@ -238,6 +711,10 @@ int main(int argc, char *argv[]) signal(SIGINT, sig_handler); signal(SIGTERM, sig_handler); +#if defined(RENDERDOC_SUPPORT_VULKAN) + VerifyVulkanLayer(argc, argv); +#endif + // add compiled-in support to version line { string support = "APIs supported at compile-time: ";