diff --git a/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp b/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp index a02fe0aa9..6558c77e7 100644 --- a/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp +++ b/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp @@ -294,6 +294,19 @@ void CaptureDialog::on_vulkanLayerWarn_clicked() const bool registerAll = (flags & eVulkan_RegisterAll); const bool updateAllowed = (flags & eVulkan_UpdateAllowed); + if(flags & eVulkan_Unfixable) + { + QString msg = + tr("There is an unfixable problem with your vulkan layer configuration. Please consult the " + "RenderDoc documentation, or package/distribution documentation on linux\n\n"); + + for(const rdctype::str &j : otherJSONs) + msg += ToQStr(j) + "\n"; + + RDDialog::critical(this, tr("Unfixable vulkan layer configuration"), msg); + return; + } + QString msg = tr("Vulkan capture happens through the API's layer mechanism. RenderDoc has detected that "); diff --git a/qrenderdoc/Windows/Dialogs/CaptureDialog.ui b/qrenderdoc/Windows/Dialogs/CaptureDialog.ui index 47fff0234..772f6fe31 100644 --- a/qrenderdoc/Windows/Dialogs/CaptureDialog.ui +++ b/qrenderdoc/Windows/Dialogs/CaptureDialog.ui @@ -271,7 +271,7 @@ 0 - 36 + 40 diff --git a/renderdoc/api/replay/renderdoc_replay.h b/renderdoc/api/replay/renderdoc_replay.h index 34cfbd986..5f1857d44 100644 --- a/renderdoc/api/replay/renderdoc_replay.h +++ b/renderdoc/api/replay/renderdoc_replay.h @@ -648,7 +648,7 @@ extern "C" RENDERDOC_API uint32_t RENDERDOC_CC RENDERDOC_InjectIntoProcess( extern "C" RENDERDOC_API bool RENDERDOC_CC RENDERDOC_NeedVulkanLayerRegistration( uint32_t *flags, rdctype::array *myJSONs, rdctype::array *otherJSONs); -extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_UpdateVulkanLayerRegistration(bool elevate); +extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_UpdateVulkanLayerRegistration(bool systemLevel); ////////////////////////////////////////////////////////////////////////// // Miscellaneous! diff --git a/renderdoc/api/replay/replay_enums.h b/renderdoc/api/replay/replay_enums.h index fefd27583..d38e31eaf 100644 --- a/renderdoc/api/replay/replay_enums.h +++ b/renderdoc/api/replay/replay_enums.h @@ -610,4 +610,5 @@ enum VulkanFlags eVulkan_CouldElevate = 0x8, eVulkan_RegisterAll = 0x10, eVulkan_UpdateAllowed = 0x20, + eVulkan_Unfixable = 0x40, }; diff --git a/renderdoc/core/core.cpp b/renderdoc/core/core.cpp index 2244b75da..e116ffecc 100644 --- a/renderdoc/core/core.cpp +++ b/renderdoc/core/core.cpp @@ -237,6 +237,9 @@ RenderDoc::RenderDoc() m_Overlay = eRENDERDOC_Overlay_Default; + m_VulkanCheck = NULL; + m_VulkanInstall = NULL; + m_TargetControlThreadShutdown = false; m_ControlClientThreadShutdown = false; } diff --git a/renderdoc/core/core.h b/renderdoc/core/core.h index 0ffbea1a0..bca547094 100644 --- a/renderdoc/core/core.h +++ b/renderdoc/core/core.h @@ -171,6 +171,10 @@ class IReplayDriver; typedef ReplayCreateStatus (*RemoteDriverProvider)(const char *logfile, IRemoteDriver **driver); typedef ReplayCreateStatus (*ReplayDriverProvider)(const char *logfile, IReplayDriver **driver); +typedef bool (*VulkanLayerCheck)(uint32_t &flags, std::vector &myJSONs, + std::vector &otherJSONs); +typedef void (*VulkanLayerInstall)(bool systemLevel); + typedef void (*ShutdownFunction)(); // this class mediates everything and owns any 'global' resources such as the crash handler. @@ -240,6 +244,23 @@ public: void RegisterReplayProvider(RDCDriver driver, const char *name, ReplayDriverProvider provider); void RegisterRemoteProvider(RDCDriver driver, const char *name, RemoteDriverProvider provider); + void SetVulkanLayerCheck(VulkanLayerCheck callback) { m_VulkanCheck = callback; } + void SetVulkanLayerInstall(VulkanLayerInstall callback) { m_VulkanInstall = callback; } + bool NeedVulkanLayerRegistration(uint32_t &flags, std::vector &myJSONs, + std::vector &otherJSONs) + { + if(m_VulkanCheck) + return m_VulkanCheck(flags, myJSONs, otherJSONs); + + return false; + } + + void UpdateVulkanLayerRegistration(bool systemLevel) + { + if(m_VulkanInstall) + m_VulkanInstall(systemLevel); + } + ReplayCreateStatus CreateReplayDriver(RDCDriver driverType, const char *logfile, IReplayDriver **driver); ReplayCreateStatus CreateRemoteDriver(RDCDriver driverType, const char *logfile, @@ -355,6 +376,9 @@ private: map m_ReplayDriverProviders; map m_RemoteDriverProviders; + VulkanLayerCheck m_VulkanCheck; + VulkanLayerInstall m_VulkanInstall; + set m_ShutdownFunctions; struct FrameCap diff --git a/renderdoc/driver/vulkan/vk_android.cpp b/renderdoc/driver/vulkan/vk_android.cpp index 0c9538d45..b5947ee10 100644 --- a/renderdoc/driver/vulkan/vk_android.cpp +++ b/renderdoc/driver/vulkan/vk_android.cpp @@ -57,3 +57,14 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h } const char *VulkanLibraryName = "libvulkan.so"; + +bool VulkanReplay::CheckVulkanLayer(uint32_t &flags, std::vector &myJSONs, + std::vector &otherJSONs) +{ + // nothing to do + return false; +} + +void VulkanReplay::InstallVulkanLayer(bool systemLevel) +{ +} diff --git a/renderdoc/driver/vulkan/vk_apple.cpp b/renderdoc/driver/vulkan/vk_apple.cpp index 85764094a..7e570155f 100644 --- a/renderdoc/driver/vulkan/vk_apple.cpp +++ b/renderdoc/driver/vulkan/vk_apple.cpp @@ -42,3 +42,13 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h } const char *VulkanLibraryName = "libvulkan.so"; + +bool VulkanReplay::CheckVulkanLayer(uint32_t &flags, std::vector &myJSONs, + std::vector &otherJSONs) +{ + return false; +} + +void VulkanReplay::InstallVulkanLayer(bool systemLevel) +{ +} diff --git a/renderdoc/driver/vulkan/vk_linux.cpp b/renderdoc/driver/vulkan/vk_linux.cpp index a48b1a5c0..05a568e27 100644 --- a/renderdoc/driver/vulkan/vk_linux.cpp +++ b/renderdoc/driver/vulkan/vk_linux.cpp @@ -23,6 +23,10 @@ ******************************************************************************/ #include "api/replay/renderdoc_replay.h" +#include "serialise/string_utils.h" + +#include +#include #include "vk_core.h" #include "vk_replay.h" @@ -141,8 +145,311 @@ const char *VulkanLibraryName = "libvulkan.so.1"; extern unsigned char driver_vulkan_renderdoc_json[]; extern int driver_vulkan_renderdoc_json_len; -extern "C" __attribute__((visibility("default"))) void RENDERDOC_GetLayerJSON(char **txt, int *len) +static std::string GenerateJSON(const std::string &sopath) { - *txt = (char *)driver_vulkan_renderdoc_json; - *len = driver_vulkan_renderdoc_json_len; + char *txt = (char *)driver_vulkan_renderdoc_json; + int len = driver_vulkan_renderdoc_json_len; + + string json = string(txt, txt + len); + + const char dllPathString[] = ".\\\\renderdoc.dll"; + + size_t idx = json.find(dllPathString); + + return json.substr(0, idx) + sopath + json.substr(idx + sizeof(dllPathString) - 1); } + +static bool FileExists(const std::string &path) +{ + return access(path.c_str(), F_OK) == 0; +} + +static std::string GetSOFromJSON(const std::string &json) +{ + char *json_string = new char[1024]; + memset(json_string, 0, 1024); + + FILE *f = fopen(json.c_str(), "r"); + + if(f) + { + fread(json_string, 1, 1024, f); + + fclose(f); + } + + string ret = ""; + + // The line is: + // "library_path": "/foo/bar/librenderdoc.so", + char *c = strstr(json_string, "library_path"); + + if(c) + { + c += sizeof("library_path\": \"") - 1; + + char *quote = strchr(c, '"'); + + if(quote) + { + *quote = 0; + ret = c; + } + } + + delete[] json_string; + + return ret; +} + +enum +{ + USR, + ETC, + HOME, + COUNT +}; + +string layerRegistrationPath[COUNT] = { + "/usr/share/vulkan/implicit_layer.d/renderdoc_capture.json", + "/etc/vulkan/implicit_layer.d/renderdoc_capture.json", + string(getenv("HOME")) + "/.local/share/vulkan/implicit_layer.d/renderdoc_capture.json"}; + +string GetThisLibPath() +{ + string librenderdoc_path; + + FILE *f = fopen("/proc/self/maps", "r"); + + if(f) + { + // read the whole thing in one go. There's no need to try and be tight with + // this allocation, so just make sure we can read everything. + char *map_string = new char[1024 * 1024]; + memset(map_string, 0, 1024 * 1024); + + fread(map_string, 1, 1024 * 1024, f); + + fclose(f); + + char *c = strstr(map_string, "/librenderdoc.so"); + + if(c) + { + // walk backwards until we hit the start of the line + while(c > map_string) + { + c--; + + if(c[0] == '\n') + { + c++; + break; + } + } + + // walk forwards across the address range (00400000-0040c000) + while(isalnum(c[0]) || c[0] == '-') + c++; + + // whitespace + while(c[0] == ' ') + c++; + + // permissions (r-xp) + while(isalpha(c[0]) || c[0] == '-') + c++; + + // whitespace + while(c[0] == ' ') + c++; + + // offset (0000b000) + while(isalnum(c[0]) || c[0] == '-') + c++; + + // whitespace + while(c[0] == ' ') + c++; + + // dev + while(isdigit(c[0]) || c[0] == ':') + c++; + + // whitespace + while(c[0] == ' ') + c++; + + // inode + while(isdigit(c[0])) + c++; + + // whitespace + while(c[0] == ' ') + c++; + + // FINALLY we are at the start of the actual path + char *end = strchr(c, '\n'); + + if(end) + librenderdoc_path = string(c, end - c); + } + + delete[] map_string; + } + + return librenderdoc_path; +} + +void MakeParentDirs(std::string file) +{ + std::string dir = dirname(file); + + if(dir == "/" || dir.empty()) + return; + + MakeParentDirs(dir); + + if(FileExists(dir)) + return; + + mkdir(dir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); +} + +bool VulkanReplay::CheckVulkanLayer(uint32_t &flags, std::vector &myJSONs, + std::vector &otherJSONs) +{ + // see if the user has suppressed all this checking as a "I know what I'm doing" measure + + if(FileExists(string(getenv("HOME")) + "/.renderdoc/ignore_vulkan_layer_issues")) + { + flags = eVulkan_ThisInstallRegistered; + return false; + } + + //////////////////////////////////////////////////////////////////////////////////////// + // check that there's only one layer registered, and it points to the same .so file that + // we are running with in this instance of renderdoccmd + + // this is a hack, but the only reliable way to find the absolute path to the library. + // dladdr would be fine but it returns the wrong result for symbols in the library + string librenderdoc_path = GetThisLibPath(); + + // it's impractical to determine whether the currently running RenderDoc build is just a loose + // extract of a tarball or a distribution that decided to put all the files in the same folder, + // and whether or not the library is in ld's searchpath. + // + // Instead we just make the requirement that renderdoc.json will always contain an absolute path + // to the matching librenderdoc.so, so that we can check if it points to this build or another + // build etc. + // + // Note there are three places to register layers - /usr, /etc and /home. The first is reserved + // for distribution packages, so if it conflicts or needs to be deleted for this install to run, + // we can't do that and have to just prompt the user. /etc we can mess with since that's for + // non-distribution packages, but it will need root permissions. + + bool exist[COUNT]; + bool match[COUNT]; + + int numExist = 0; + int numMatch = 0; + + for(int i = 0; i < COUNT; i++) + { + exist[i] = FileExists(layerRegistrationPath[i]); + match[i] = (GetSOFromJSON(layerRegistrationPath[i]) == librenderdoc_path); + + if(exist[i]) + numExist++; + + if(match[i]) + numMatch++; + } + + flags = eVulkan_CouldElevate | eVulkan_UpdateAllowed; + + if(numMatch >= 1) + flags |= eVulkan_ThisInstallRegistered; + + // if we only have one registration, check that it points to us. If so, we're good + if(numExist == 1 && numMatch == 1) + return false; + + if(exist[USR] && !match[USR]) + otherJSONs.push_back(layerRegistrationPath[USR]); + + if(exist[ETC] && !match[ETC]) + otherJSONs.push_back(layerRegistrationPath[ETC]); + + if(exist[HOME] && !match[HOME]) + otherJSONs.push_back(layerRegistrationPath[HOME]); + + if(!otherJSONs.empty()) + flags |= eVulkan_OtherInstallsRegistered; + + if(exist[USR] && match[USR]) + { + // just need to unregister others + } + else + { + myJSONs.push_back(layerRegistrationPath[ETC]); + myJSONs.push_back(layerRegistrationPath[HOME]); + } + + if(exist[USR] && !match[USR]) + { + flags = eVulkan_Unfixable | eVulkan_OtherInstallsRegistered; + otherJSONs.clear(); + otherJSONs.push_back(layerRegistrationPath[USR]); + } + + return true; +} + +void VulkanReplay::InstallVulkanLayer(bool systemLevel) +{ + // if we want to install to the system and there's a registration in $HOME, delete it + if(systemLevel && FileExists(layerRegistrationPath[HOME])) + { + if(unlink(layerRegistrationPath[HOME].c_str()) < 0) + { + const char *const errtext = strerror(errno); + RDCERR("Error removing %s: %s", layerRegistrationPath[HOME].c_str(), errtext); + } + } + + // and vice-versa + if(!systemLevel && FileExists(layerRegistrationPath[ETC])) + { + if(unlink(layerRegistrationPath[ETC].c_str()) < 0) + { + const char *const errtext = strerror(errno); + RDCERR("Error removing %s: %s", layerRegistrationPath[ETC].c_str(), errtext); + } + } + + int idx = systemLevel ? ETC : HOME; + + string path = GetSOFromJSON(layerRegistrationPath[idx]); + string libPath = GetThisLibPath(); + + if(path != libPath) + { + MakeParentDirs(layerRegistrationPath[idx]); + + FILE *f = fopen(layerRegistrationPath[idx].c_str(), "w"); + + if(f) + { + fputs(GenerateJSON(libPath).c_str(), f); + + fclose(f); + } + else + { + const char *const errtext = strerror(errno); + RDCERR("Error writing %s: %s", layerRegistrationPath[idx].c_str(), errtext); + } + } +} \ No newline at end of file diff --git a/renderdoc/driver/vulkan/vk_replay.cpp b/renderdoc/driver/vulkan/vk_replay.cpp index e6e07f89b..65f965566 100644 --- a/renderdoc/driver/vulkan/vk_replay.cpp +++ b/renderdoc/driver/vulkan/vk_replay.cpp @@ -5630,4 +5630,14 @@ ReplayCreateStatus Vulkan_CreateReplayDevice(const char *logfile, IReplayDriver return eReplayCreate_Success; } -static DriverRegistration VkDriverRegistration(RDC_Vulkan, "Vulkan", &Vulkan_CreateReplayDevice); +struct VulkanDriverRegistration +{ + VulkanDriverRegistration() + { + RenderDoc::Inst().RegisterReplayProvider(RDC_Vulkan, "Vulkan", &Vulkan_CreateReplayDevice); + RenderDoc::Inst().SetVulkanLayerCheck(&VulkanReplay::CheckVulkanLayer); + RenderDoc::Inst().SetVulkanLayerInstall(&VulkanReplay::InstallVulkanLayer); + } +}; + +static VulkanDriverRegistration VkDriverRegistration; diff --git a/renderdoc/driver/vulkan/vk_replay.h b/renderdoc/driver/vulkan/vk_replay.h index 022a03ff1..2298ab662 100644 --- a/renderdoc/driver/vulkan/vk_replay.h +++ b/renderdoc/driver/vulkan/vk_replay.h @@ -258,6 +258,14 @@ public: // called before the VkDevice is destroyed, to shutdown any counters void PreDeviceShutdownCounters(); + // used for vulkan layer bookkeeping. Ideally this should all be handled by installers/packages, + // but for developers running builds locally or just in case, we need to be able to update the + // layer registration ourselves. + // These functions are defined in vk_.cpp + static bool CheckVulkanLayer(uint32_t &flags, std::vector &myJSONs, + std::vector &otherJSONs); + static void InstallVulkanLayer(bool systemLevel); + private: struct OutputWindow { diff --git a/renderdoc/driver/vulkan/vk_win32.cpp b/renderdoc/driver/vulkan/vk_win32.cpp index a0657d0ae..37caa94fd 100644 --- a/renderdoc/driver/vulkan/vk_win32.cpp +++ b/renderdoc/driver/vulkan/vk_win32.cpp @@ -157,3 +157,193 @@ VkBool32 WrappedVulkan::vkGetPhysicalDeviceWin32PresentationSupportKHR(VkPhysica } const char *VulkanLibraryName = "vulkan-1.dll"; + +std::wstring GetJSONPath(bool wow6432) +{ + wchar_t curFile[1024]; + GetModuleFileNameW(NULL, curFile, 1024); + + wchar_t *lastSlash = wcsrchr(curFile, '\\'); + if(lastSlash) + *(lastSlash + 1) = 0; + + if(wow6432) + wcscat_s(curFile, L"x86\\"); + + wcscat_s(curFile, L"renderdoc.json"); + + return curFile; +} + +static HKEY GetImplicitLayersKey(bool writeable, bool wow6432) +{ + std::string basepath = "SOFTWARE\\"; + + if(wow6432) + basepath += "Wow6432Node\\"; + + basepath += "Khronos\\Vulkan\\ImplicitLayers"; + + HKEY key = NULL; + LSTATUS ret = ERROR_SUCCESS; + + if(writeable) + ret = RegCreateKeyExA(HKEY_LOCAL_MACHINE, basepath.c_str(), 0, NULL, 0, KEY_READ | KEY_WRITE, + NULL, &key, NULL); + else + ret = RegOpenKeyExA(HKEY_LOCAL_MACHINE, basepath.c_str(), 0, KEY_READ, &key); + + if(ret != ERROR_SUCCESS) + { + if(key) + RegCloseKey(key); + + // find to fail to open for read, the key may not exist + if(writeable) + RDCERR("Couldn't open %s for write", basepath.c_str()); + + return NULL; + } + + return key; +} + +bool ProcessImplicitLayersKey(HKEY key, const std::wstring &path, + std::vector *otherJSONs, bool deleteOthers) +{ + bool thisRegistered = false; + + wchar_t name[1024] = {}; + DWORD nameSize = 1024; + DWORD idx = 0; + + LONG ret = RegEnumValueW(key, idx++, name, &nameSize, NULL, NULL, NULL, NULL); + + std::wstring myJSON = path; + for(size_t i = 0; i < myJSON.size(); i++) + myJSON[i] = towlower(myJSON[i]); + + while(ret == ERROR_SUCCESS) + { + // convert the name here so we preserve casing + std::string utf8name = StringFormat::Wide2UTF8(name); + + for(DWORD i = 0; i <= nameSize && name[i]; i++) + name[i] = towlower(name[i]); + + if(wcscmp(name, myJSON.c_str()) == 0) + { + thisRegistered = true; + } + else if(wcsstr(name, L"renderdoc.json") != NULL) + { + if(otherJSONs) + otherJSONs->push_back(utf8name); + + if(deleteOthers) + RegDeleteValueW(key, name); + } + + nameSize = 1024; + ret = RegEnumValueW(key, idx++, name, &nameSize, NULL, NULL, NULL, NULL); + } + + return thisRegistered; +} + +bool VulkanReplay::CheckVulkanLayer(uint32_t &flags, std::vector &myJSONs, + std::vector &otherJSONs) +{ + std::wstring normalPath = GetJSONPath(false); + myJSONs.push_back(StringFormat::Wide2UTF8(normalPath)); + +#if ENABLED(RDOC_X64) + std::wstring wow6432Path = GetJSONPath(true); + myJSONs.push_back(StringFormat::Wide2UTF8(wow6432Path)); +#endif + + HKEY key = GetImplicitLayersKey(false, false); + + // if we couldn't even get the ImplicitLayers reg key the system doesn't have the + // vulkan runtime, so we return as if we are not registered (as that's the case). + // People not using vulkan can either ignore the message, or click to set it up + // and it will go away as we'll have rights to create it. + if(!key) + { + flags = eVulkan_NeedElevation | eVulkan_RegisterAll; + return true; + } + + bool thisRegistered = ProcessImplicitLayersKey(key, normalPath, &otherJSONs, false); + + RegCloseKey(key); + +#if ENABLED(RDOC_X64) + { + key = GetImplicitLayersKey(false, true); + + // if we're on 64-bit, the layer isn't registered unless both keys are registered. + thisRegistered = false; + + if(key) + { + thisRegistered = ProcessImplicitLayersKey(key, wow6432Path, &otherJSONs, false); + + RegCloseKey(key); + } + } +#endif + + flags = eVulkan_NeedElevation | eVulkan_RegisterAll; + + if(thisRegistered) + flags |= eVulkan_ThisInstallRegistered; + + if(!otherJSONs.empty()) + flags |= eVulkan_OtherInstallsRegistered; + + // return true if any changes are needed + return !otherJSONs.empty() || !thisRegistered; +} + +void VulkanReplay::InstallVulkanLayer(bool systemLevel) +{ + HKEY key = GetImplicitLayersKey(true, false); + + const DWORD zero = 0; + + if(key) + { + std::wstring path = GetJSONPath(false); + + // this function will delete all non-matching renderdoc.json values, and return true if our own + // is registered + bool thisRegistered = ProcessImplicitLayersKey(key, path, NULL, true); + + if(!thisRegistered) + RegSetValueExW(key, path.c_str(), 0, REG_DWORD, (const BYTE *)&zero, sizeof(zero)); + + RegCloseKey(key); + } + +// if we're a 64-bit process, update the 32-bit key +#if ENABLED(RDOC_X64) + { + HKEY key = GetImplicitLayersKey(true, true); + + if(key) + { + std::wstring path = GetJSONPath(true); + + // this function will delete all non-matching renderdoc.json values, and return true if our + // own is registered + bool thisRegistered = ProcessImplicitLayersKey(key, path, NULL, true); + + if(!thisRegistered) + RegSetValueExW(key, path.c_str(), 0, REG_DWORD, (const BYTE *)&zero, sizeof(zero)); + + RegCloseKey(key); + } + } +#endif +} diff --git a/renderdoc/replay/entry_points.cpp b/renderdoc/replay/entry_points.cpp index 21fc01959..baef0be4c 100644 --- a/renderdoc/replay/entry_points.cpp +++ b/renderdoc/replay/entry_points.cpp @@ -799,15 +799,37 @@ extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StartAndroidRemoteServer() "shell am start -n org.renderdoc.renderdoccmd/.Loader -e renderdoccmd remoteserver"); } -extern "C" RENDERDOC_API bool RENDERDOC_CC RENDERDOC_NeedVulkanLayerRegistration( - uint32_t *flags, rdctype::array *myJSONs, rdctype::array *otherJSONs) +extern "C" RENDERDOC_API bool RENDERDOC_CC +RENDERDOC_NeedVulkanLayerRegistration(uint32_t *flagsPtr, rdctype::array *myJSONsPtr, + rdctype::array *otherJSONsPtr) { - // stub + uint32_t flags = 0; + std::vector myJSONs; + std::vector otherJSONs; - return false; + bool ret = RenderDoc::Inst().NeedVulkanLayerRegistration(flags, myJSONs, otherJSONs); + + if(flagsPtr) + *flagsPtr = flags; + + if(myJSONsPtr) + { + create_array(*myJSONsPtr, myJSONs.size()); + for(size_t i = 0; i < myJSONs.size(); i++) + (*myJSONsPtr)[i] = myJSONs[i]; + } + + if(otherJSONsPtr) + { + create_array(*otherJSONsPtr, otherJSONs.size()); + for(size_t i = 0; i < otherJSONs.size(); i++) + (*otherJSONsPtr)[i] = otherJSONs[i]; + } + + return ret; } -extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_UpdateVulkanLayerRegistration(bool elevate) +extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_UpdateVulkanLayerRegistration(bool systemLevel) { - // stub + RenderDoc::Inst().UpdateVulkanLayerRegistration(systemLevel); } \ No newline at end of file diff --git a/renderdoccmd/renderdoccmd_linux.cpp b/renderdoccmd/renderdoccmd_linux.cpp index 6e298f4c3..6f221d4b2 100644 --- a/renderdoccmd/renderdoccmd_linux.cpp +++ b/renderdoccmd/renderdoccmd_linux.cpp @@ -51,124 +51,15 @@ void Daemonise() daemon(1, 0); } -// this is exported from vk_linux.cpp - -#if defined(RENDERDOC_SUPPORT_VULKAN) - -extern "C" void RENDERDOC_GetLayerJSON(char **txt, int *len); - -#else - -// just for ease of compiling, define a dummy function - -void RENDERDOC_GetLayerJSON(char **txt, int *len) -{ - static char dummy[] = ""; - *txt = dummy; - *len = 0; -} - -#endif - -static string GenerateJSON(const string &sopath) -{ - char *txt = NULL; - int len = 0; - - RENDERDOC_GetLayerJSON(&txt, &len); - - if(len <= 0) - return ""; - - string json = string(txt, txt + len); - - const char dllPathString[] = ".\\\\renderdoc.dll"; - - size_t idx = json.find(dllPathString); - - return json.substr(0, idx) + sopath + json.substr(idx + sizeof(dllPathString) - 1); -} - -static bool FileExists(const string &path) -{ - FILE *f = fopen(path.c_str(), "r"); - - if(f) - { - fclose(f); - return true; - } - - return false; -} - -static string GetSOFromJSON(const string &json) -{ - char *json_string = new char[1024]; - memset(json_string, 0, 1024); - - FILE *f = fopen(json.c_str(), "r"); - - if(f) - { - fread(json_string, 1, 1024, f); - - fclose(f); - } - - string ret = ""; - - // The line is: - // "library_path": "/foo/bar/librenderdoc.so", - char *c = strstr(json_string, "library_path"); - - if(c) - { - c += sizeof("library_path\": \"") - 1; - - char *quote = strchr(c, '"'); - - if(quote) - { - *quote = 0; - ret = c; - } - } - - delete[] json_string; - - return ret; -} - -enum -{ - USR, - ETC, - HOME, - COUNT -}; - -string layerRegistrationPath[COUNT] = { - "/usr/share/vulkan/implicit_layer.d/renderdoc_capture.json", - "/etc/vulkan/implicit_layer.d/renderdoc_capture.json", - string(getenv("HOME")) + "/.local/share/vulkan/implicit_layer.d/renderdoc_capture.json"}; - struct VulkanRegisterCommand : public Command { - VulkanRegisterCommand(bool layer_exists[COUNT], const string &path) - { - etcExists = layer_exists[ETC]; - homeExists = layer_exists[HOME]; - libPath = path; - } - + VulkanRegisterCommand() {} virtual void AddOptions(cmdline::parser &parser) { parser.add("ignore", 'i', "Do nothing and don't warn about Vulkan layer issues."); parser.add( "system", '\0', "Install layer registration to /etc instead of $HOME/.local (requires root privileges)"); - parser.add("dry-run", 'n', "Don't perform any actions, instead print what would happen."); } virtual const char *Description() { @@ -208,233 +99,24 @@ struct VulkanRegisterCommand : public Command return 0; } - bool system = (parser.exist("system")); - bool dryrun = (parser.exist("dry-run")); - - // if we want to install to the system and there's a registration in $HOME, delete it - if(system && homeExists) - { - std::cout << "Removing '" << layerRegistrationPath[HOME] << "'" << std::endl; - - if(!dryrun) - { - int ret = unlink(layerRegistrationPath[HOME].c_str()); - - if(ret < 0) - { - const char *const errtext = strerror(errno); - std::cout << "Error - " << errtext << std::endl; - } - } - } - - // and vice-versa - if(!system && etcExists) - { - std::cout << "Removing '" << layerRegistrationPath[ETC] << "'" << std::endl; - - if(!dryrun) - { - int ret = unlink(layerRegistrationPath[ETC].c_str()); - - if(ret < 0) - { - const char *const errtext = strerror(errno); - std::cout << "Error - " << errtext << std::endl; - } - } - } - - int idx = system ? ETC : HOME; - - string path = GetSOFromJSON(layerRegistrationPath[idx]); - - if(path != libPath) - { - if((system && !etcExists) || (!system && !homeExists)) - { - std::cout << "Registering '" << layerRegistrationPath[idx] << "'" << std::endl; - } - else - { - std::cout << "Updating '" << layerRegistrationPath[idx] << "'" << std::endl; - if(path == "") - { - std::cout << " JSON is corrupt or unrecognised, replacing with valid JSON pointing" - << std::endl; - std::cout << " to '" << libPath << "'" << std::endl; - } - else - { - std::cout << " Repointing from '" << path << "'" << std::endl; - std::cout << " to '" << libPath << "'" << std::endl; - } - } - - if(!dryrun) - { - FILE *f = fopen(layerRegistrationPath[idx].c_str(), "w"); - - if(f) - { - fputs(GenerateJSON(libPath).c_str(), f); - - fclose(f); - } - else - { - const char *const errtext = strerror(errno); - std::cout << "Error - " << errtext << std::endl; - } - } - } + RENDERDOC_UpdateVulkanLayerRegistration(parser.exist("system")); return 0; } - - bool etcExists; - bool homeExists; - string libPath; }; void VerifyVulkanLayer(int argc, char *argv[]) { - // see if the user has suppressed all this checking as a "I know what I'm doing" measure + uint32_t flags = 0; + rdctype::array myJSONs; + rdctype::array otherJSONs; - string ignorePath = string(getenv("HOME")) + "/.renderdoc/ignore_vulkan_layer_issues"; - if(FileExists(ignorePath)) - return; - - //////////////////////////////////////////////////////////////////////////////////////// - // check that there's only one layer registered, and it points to the same .so file that - // we are running with in this instance of renderdoccmd - - // this is a hack, but the only reliable way to find the absolute path to the library. - // dladdr would be fine but it returns the wrong result for symbols in the library - string librenderdoc_path; + bool needUpdate = RENDERDOC_NeedVulkanLayerRegistration(&flags, &myJSONs, &otherJSONs); + if(!needUpdate) { - FILE *f = fopen("/proc/self/maps", "r"); - - if(f) - { - // read the whole thing in one go. There's no need to try and be tight with - // this allocation, so just make sure we can read everything. - char *map_string = new char[1024 * 1024]; - memset(map_string, 0, 1024 * 1024); - - fread(map_string, 1, 1024 * 1024, f); - - fclose(f); - - char *c = strstr(map_string, "/librenderdoc.so"); - - if(c) - { - // walk backwards until we hit the start of the line - while(c > map_string) - { - c--; - - if(c[0] == '\n') - { - c++; - break; - } - } - - // walk forwards across the address range (00400000-0040c000) - while(isalnum(c[0]) || c[0] == '-') - c++; - - // whitespace - while(c[0] == ' ') - c++; - - // permissions (r-xp) - while(isalpha(c[0]) || c[0] == '-') - c++; - - // whitespace - while(c[0] == ' ') - c++; - - // offset (0000b000) - while(isalnum(c[0]) || c[0] == '-') - c++; - - // whitespace - while(c[0] == ' ') - c++; - - // dev - while(isdigit(c[0]) || c[0] == ':') - c++; - - // whitespace - while(c[0] == ' ') - c++; - - // inode - while(isdigit(c[0])) - c++; - - // whitespace - while(c[0] == ' ') - c++; - - // FINALLY we are at the start of the actual path - char *end = strchr(c, '\n'); - - if(end) - { - librenderdoc_path = string(c, end - c); - } - } - - delete[] map_string; - } - } - - // it's impractical to determine whether the currently running RenderDoc build is just a loose - // extract of a tarball or a distribution that decided to put all the files in the same folder, - // and whether or not the library is in ld's searchpath. - // - // Instead we just make the requirement that renderdoc.json will always contain an absolute path - // to the matching librenderdoc.so, so that we can check if it points to this build or another - // build etc. - // - // Note there are three places to register layers - /usr, /etc and /home. The first is reserved - // for distribution packages, so if it conflicts or needs to be deleted for this install to run, - // we can't do that and have to just prompt the user. /etc we can mess with since that's for - // non-distribution packages, but it will need root permissions. - - bool exist[COUNT]; - bool match[COUNT]; - - int numExist = 0; - int numMatch = 0; - - for(int i = 0; i < COUNT; i++) - { - exist[i] = FileExists(layerRegistrationPath[i]); - match[i] = (GetSOFromJSON(layerRegistrationPath[i]) == librenderdoc_path); - - if(exist[i]) - numExist++; - - if(match[i]) - numMatch++; - } - - // if we only have one registration, check that it points to us. If so, we're good - if(numExist == 1 && numMatch == 1) - return; - - // if we're about to execute the command, don't print all this explanatory text. - if(argc > 1 && !strcmp(argv[1], "vulkanregister")) - { - add_command("vulkanregister", new VulkanRegisterCommand(exist, librenderdoc_path)); + if(!(flags & eVulkan_Unfixable)) + add_command("vulkanregister", new VulkanRegisterCommand()); return; } @@ -443,60 +125,40 @@ void VerifyVulkanLayer(int argc, char *argv[]) std::cerr << "** Warning: Vulkan capture possibly not configured. **" << std::endl; std::cerr << std::endl; - if(numExist > 1) + + if(flags & eVulkan_OtherInstallsRegistered) std::cerr << "Multiple RenderDoc layers are registered, possibly from different builds." << std::endl; - else if(numExist < 0) - std::cerr << "RenderDoc layer is not registered." << std::endl; - else - std::cerr << "RenderDoc layer is registered, but to a different library." << std::endl; + + if(!(flags & eVulkan_ThisInstallRegistered)) + std::cerr << "This build's RenderDoc layer is not registered." << std::endl; + std::cerr << "To fix this, the following actions must take place: " << std::endl << std::endl; - bool printed = false; + const bool registerAll = (flags & eVulkan_RegisterAll); + const bool updateAllowed = (flags & eVulkan_UpdateAllowed); - if(exist[USR] && !match[USR]) - { - std::cerr << "* Unregister: '" << layerRegistrationPath[USR] << "'" << std::endl; - printed = true; - } + for(const rdctype::str &j : otherJSONs) + std::cerr << (updateAllowed ? "Unregister/update: " : "Unregister: ") << j.c_str() << std::endl; - if(exist[ETC] && !match[ETC]) + if(!(flags & eVulkan_ThisInstallRegistered)) { - std::cerr << "* Unregister or update: '" << layerRegistrationPath[ETC] << "'" << std::endl; - printed = true; - } - - if(exist[HOME] && !match[HOME]) - { - std::cerr << "* Unregister or update: '" << layerRegistrationPath[HOME] << "'" << std::endl; - printed = true; - } - - if(printed) - std::cerr << std::endl; - - if(exist[USR] && match[USR]) - { - // just need to unregister others - } - else - { - if(!exist[ETC] && !exist[HOME]) + if(registerAll) { - std::cerr << "* Register either: '" << layerRegistrationPath[ETC] << "'" << std::endl; - std::cerr << " or: '" << layerRegistrationPath[HOME] << "'" << std::endl; + for(const rdctype::str &j : myJSONs) + std::cerr << (updateAllowed ? "Register/update: " : "Register: ") << j.c_str() << std::endl; } else { - std::cerr << "* Update or register either: '" << layerRegistrationPath[ETC] << "'" << std::endl; - std::cerr << " or: '" << layerRegistrationPath[HOME] << "'" - << std::endl; + std::cerr << (updateAllowed ? "Register one of:" : "Register/update one of:") << std::endl; + for(const rdctype::str &j : myJSONs) + std::cerr << " -- " << j.c_str() << "\n"; } - - std::cerr << std::endl; } - if(exist[USR] && !match[USR]) + std::cerr << std::endl; + + if(flags & eVulkan_Unfixable) { std::cerr << "NOTE: The renderdoc layer registered in /usr is reserved for distribution" << std::endl; @@ -535,7 +197,7 @@ void VerifyVulkanLayer(int argc, char *argv[]) << std::endl; std::cerr << std::endl; - add_command("vulkanregister", new VulkanRegisterCommand(exist, librenderdoc_path)); + add_command("vulkanregister", new VulkanRegisterCommand()); } void DisplayRendererPreview(ReplayRenderer *renderer, TextureDisplay &displayCfg, uint32_t width,