mirror of
https://github.com/baldurk/renderdoc.git
synced 2026-05-04 09:00:44 +00:00
Implement win32/linux checking and registering of vulkan layers
This commit is contained in:
baldurk
@@ -294,6 +294,19 @@ void CaptureDialog::on_vulkanLayerWarn_clicked()
|
||||
const bool registerAll = (flags & eVulkan_RegisterAll);
|
||||
const bool updateAllowed = (flags & eVulkan_UpdateAllowed);
|
||||
|
||||
if(flags & eVulkan_Unfixable)
|
||||
{
|
||||
QString msg =
|
||||
tr("There is an unfixable problem with your vulkan layer configuration. Please consult the "
|
||||
"RenderDoc documentation, or package/distribution documentation on linux\n\n");
|
||||
|
||||
for(const rdctype::str &j : otherJSONs)
|
||||
msg += ToQStr(j) + "\n";
|
||||
|
||||
RDDialog::critical(this, tr("Unfixable vulkan layer configuration"), msg);
|
||||
return;
|
||||
}
|
||||
|
||||
QString msg =
|
||||
tr("Vulkan capture happens through the API's layer mechanism. RenderDoc has detected that ");
|
||||
|
||||
|
||||
@@ -271,7 +271,7 @@
|
||||
<property name="minimumSize">
|
||||
<size>
|
||||
<width>0</width>
|
||||
<height>36</height>
|
||||
<height>40</height>
|
||||
</size>
|
||||
</property>
|
||||
<property name="cursor">
|
||||
|
||||
@@ -648,7 +648,7 @@ extern "C" RENDERDOC_API uint32_t RENDERDOC_CC RENDERDOC_InjectIntoProcess(
|
||||
|
||||
extern "C" RENDERDOC_API bool RENDERDOC_CC RENDERDOC_NeedVulkanLayerRegistration(
|
||||
uint32_t *flags, rdctype::array<rdctype::str> *myJSONs, rdctype::array<rdctype::str> *otherJSONs);
|
||||
extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_UpdateVulkanLayerRegistration(bool elevate);
|
||||
extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_UpdateVulkanLayerRegistration(bool systemLevel);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// Miscellaneous!
|
||||
|
||||
@@ -610,4 +610,5 @@ enum VulkanFlags
|
||||
eVulkan_CouldElevate = 0x8,
|
||||
eVulkan_RegisterAll = 0x10,
|
||||
eVulkan_UpdateAllowed = 0x20,
|
||||
eVulkan_Unfixable = 0x40,
|
||||
};
|
||||
|
||||
@@ -237,6 +237,9 @@ RenderDoc::RenderDoc()
|
||||
|
||||
m_Overlay = eRENDERDOC_Overlay_Default;
|
||||
|
||||
m_VulkanCheck = NULL;
|
||||
m_VulkanInstall = NULL;
|
||||
|
||||
m_TargetControlThreadShutdown = false;
|
||||
m_ControlClientThreadShutdown = false;
|
||||
}
|
||||
|
||||
@@ -171,6 +171,10 @@ class IReplayDriver;
|
||||
typedef ReplayCreateStatus (*RemoteDriverProvider)(const char *logfile, IRemoteDriver **driver);
|
||||
typedef ReplayCreateStatus (*ReplayDriverProvider)(const char *logfile, IReplayDriver **driver);
|
||||
|
||||
typedef bool (*VulkanLayerCheck)(uint32_t &flags, std::vector<std::string> &myJSONs,
|
||||
std::vector<std::string> &otherJSONs);
|
||||
typedef void (*VulkanLayerInstall)(bool systemLevel);
|
||||
|
||||
typedef void (*ShutdownFunction)();
|
||||
|
||||
// this class mediates everything and owns any 'global' resources such as the crash handler.
|
||||
@@ -240,6 +244,23 @@ public:
|
||||
void RegisterReplayProvider(RDCDriver driver, const char *name, ReplayDriverProvider provider);
|
||||
void RegisterRemoteProvider(RDCDriver driver, const char *name, RemoteDriverProvider provider);
|
||||
|
||||
void SetVulkanLayerCheck(VulkanLayerCheck callback) { m_VulkanCheck = callback; }
|
||||
void SetVulkanLayerInstall(VulkanLayerInstall callback) { m_VulkanInstall = callback; }
|
||||
bool NeedVulkanLayerRegistration(uint32_t &flags, std::vector<std::string> &myJSONs,
|
||||
std::vector<std::string> &otherJSONs)
|
||||
{
|
||||
if(m_VulkanCheck)
|
||||
return m_VulkanCheck(flags, myJSONs, otherJSONs);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
void UpdateVulkanLayerRegistration(bool systemLevel)
|
||||
{
|
||||
if(m_VulkanInstall)
|
||||
m_VulkanInstall(systemLevel);
|
||||
}
|
||||
|
||||
ReplayCreateStatus CreateReplayDriver(RDCDriver driverType, const char *logfile,
|
||||
IReplayDriver **driver);
|
||||
ReplayCreateStatus CreateRemoteDriver(RDCDriver driverType, const char *logfile,
|
||||
@@ -355,6 +376,9 @@ private:
|
||||
map<RDCDriver, ReplayDriverProvider> m_ReplayDriverProviders;
|
||||
map<RDCDriver, RemoteDriverProvider> m_RemoteDriverProviders;
|
||||
|
||||
VulkanLayerCheck m_VulkanCheck;
|
||||
VulkanLayerInstall m_VulkanInstall;
|
||||
|
||||
set<ShutdownFunction> m_ShutdownFunctions;
|
||||
|
||||
struct FrameCap
|
||||
|
||||
@@ -57,3 +57,14 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h
|
||||
}
|
||||
|
||||
const char *VulkanLibraryName = "libvulkan.so";
|
||||
|
||||
bool VulkanReplay::CheckVulkanLayer(uint32_t &flags, std::vector<std::string> &myJSONs,
|
||||
std::vector<std::string> &otherJSONs)
|
||||
{
|
||||
// nothing to do
|
||||
return false;
|
||||
}
|
||||
|
||||
void VulkanReplay::InstallVulkanLayer(bool systemLevel)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -42,3 +42,13 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h
|
||||
}
|
||||
|
||||
const char *VulkanLibraryName = "libvulkan.so";
|
||||
|
||||
bool VulkanReplay::CheckVulkanLayer(uint32_t &flags, std::vector<std::string> &myJSONs,
|
||||
std::vector<std::string> &otherJSONs)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
void VulkanReplay::InstallVulkanLayer(bool systemLevel)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -23,6 +23,10 @@
|
||||
******************************************************************************/
|
||||
|
||||
#include "api/replay/renderdoc_replay.h"
|
||||
#include "serialise/string_utils.h"
|
||||
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "vk_core.h"
|
||||
#include "vk_replay.h"
|
||||
@@ -141,8 +145,311 @@ const char *VulkanLibraryName = "libvulkan.so.1";
|
||||
extern unsigned char driver_vulkan_renderdoc_json[];
|
||||
extern int driver_vulkan_renderdoc_json_len;
|
||||
|
||||
extern "C" __attribute__((visibility("default"))) void RENDERDOC_GetLayerJSON(char **txt, int *len)
|
||||
static std::string GenerateJSON(const std::string &sopath)
|
||||
{
|
||||
*txt = (char *)driver_vulkan_renderdoc_json;
|
||||
*len = driver_vulkan_renderdoc_json_len;
|
||||
char *txt = (char *)driver_vulkan_renderdoc_json;
|
||||
int len = driver_vulkan_renderdoc_json_len;
|
||||
|
||||
string json = string(txt, txt + len);
|
||||
|
||||
const char dllPathString[] = ".\\\\renderdoc.dll";
|
||||
|
||||
size_t idx = json.find(dllPathString);
|
||||
|
||||
return json.substr(0, idx) + sopath + json.substr(idx + sizeof(dllPathString) - 1);
|
||||
}
|
||||
|
||||
static bool FileExists(const std::string &path)
|
||||
{
|
||||
return access(path.c_str(), F_OK) == 0;
|
||||
}
|
||||
|
||||
static std::string GetSOFromJSON(const std::string &json)
|
||||
{
|
||||
char *json_string = new char[1024];
|
||||
memset(json_string, 0, 1024);
|
||||
|
||||
FILE *f = fopen(json.c_str(), "r");
|
||||
|
||||
if(f)
|
||||
{
|
||||
fread(json_string, 1, 1024, f);
|
||||
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
string ret = "";
|
||||
|
||||
// The line is:
|
||||
// "library_path": "/foo/bar/librenderdoc.so",
|
||||
char *c = strstr(json_string, "library_path");
|
||||
|
||||
if(c)
|
||||
{
|
||||
c += sizeof("library_path\": \"") - 1;
|
||||
|
||||
char *quote = strchr(c, '"');
|
||||
|
||||
if(quote)
|
||||
{
|
||||
*quote = 0;
|
||||
ret = c;
|
||||
}
|
||||
}
|
||||
|
||||
delete[] json_string;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
enum
|
||||
{
|
||||
USR,
|
||||
ETC,
|
||||
HOME,
|
||||
COUNT
|
||||
};
|
||||
|
||||
string layerRegistrationPath[COUNT] = {
|
||||
"/usr/share/vulkan/implicit_layer.d/renderdoc_capture.json",
|
||||
"/etc/vulkan/implicit_layer.d/renderdoc_capture.json",
|
||||
string(getenv("HOME")) + "/.local/share/vulkan/implicit_layer.d/renderdoc_capture.json"};
|
||||
|
||||
string GetThisLibPath()
|
||||
{
|
||||
string librenderdoc_path;
|
||||
|
||||
FILE *f = fopen("/proc/self/maps", "r");
|
||||
|
||||
if(f)
|
||||
{
|
||||
// read the whole thing in one go. There's no need to try and be tight with
|
||||
// this allocation, so just make sure we can read everything.
|
||||
char *map_string = new char[1024 * 1024];
|
||||
memset(map_string, 0, 1024 * 1024);
|
||||
|
||||
fread(map_string, 1, 1024 * 1024, f);
|
||||
|
||||
fclose(f);
|
||||
|
||||
char *c = strstr(map_string, "/librenderdoc.so");
|
||||
|
||||
if(c)
|
||||
{
|
||||
// walk backwards until we hit the start of the line
|
||||
while(c > map_string)
|
||||
{
|
||||
c--;
|
||||
|
||||
if(c[0] == '\n')
|
||||
{
|
||||
c++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// walk forwards across the address range (00400000-0040c000)
|
||||
while(isalnum(c[0]) || c[0] == '-')
|
||||
c++;
|
||||
|
||||
// whitespace
|
||||
while(c[0] == ' ')
|
||||
c++;
|
||||
|
||||
// permissions (r-xp)
|
||||
while(isalpha(c[0]) || c[0] == '-')
|
||||
c++;
|
||||
|
||||
// whitespace
|
||||
while(c[0] == ' ')
|
||||
c++;
|
||||
|
||||
// offset (0000b000)
|
||||
while(isalnum(c[0]) || c[0] == '-')
|
||||
c++;
|
||||
|
||||
// whitespace
|
||||
while(c[0] == ' ')
|
||||
c++;
|
||||
|
||||
// dev
|
||||
while(isdigit(c[0]) || c[0] == ':')
|
||||
c++;
|
||||
|
||||
// whitespace
|
||||
while(c[0] == ' ')
|
||||
c++;
|
||||
|
||||
// inode
|
||||
while(isdigit(c[0]))
|
||||
c++;
|
||||
|
||||
// whitespace
|
||||
while(c[0] == ' ')
|
||||
c++;
|
||||
|
||||
// FINALLY we are at the start of the actual path
|
||||
char *end = strchr(c, '\n');
|
||||
|
||||
if(end)
|
||||
librenderdoc_path = string(c, end - c);
|
||||
}
|
||||
|
||||
delete[] map_string;
|
||||
}
|
||||
|
||||
return librenderdoc_path;
|
||||
}
|
||||
|
||||
void MakeParentDirs(std::string file)
|
||||
{
|
||||
std::string dir = dirname(file);
|
||||
|
||||
if(dir == "/" || dir.empty())
|
||||
return;
|
||||
|
||||
MakeParentDirs(dir);
|
||||
|
||||
if(FileExists(dir))
|
||||
return;
|
||||
|
||||
mkdir(dir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
|
||||
}
|
||||
|
||||
bool VulkanReplay::CheckVulkanLayer(uint32_t &flags, std::vector<std::string> &myJSONs,
|
||||
std::vector<std::string> &otherJSONs)
|
||||
{
|
||||
// see if the user has suppressed all this checking as a "I know what I'm doing" measure
|
||||
|
||||
if(FileExists(string(getenv("HOME")) + "/.renderdoc/ignore_vulkan_layer_issues"))
|
||||
{
|
||||
flags = eVulkan_ThisInstallRegistered;
|
||||
return false;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////
|
||||
// check that there's only one layer registered, and it points to the same .so file that
|
||||
// we are running with in this instance of renderdoccmd
|
||||
|
||||
// this is a hack, but the only reliable way to find the absolute path to the library.
|
||||
// dladdr would be fine but it returns the wrong result for symbols in the library
|
||||
string librenderdoc_path = GetThisLibPath();
|
||||
|
||||
// it's impractical to determine whether the currently running RenderDoc build is just a loose
|
||||
// extract of a tarball or a distribution that decided to put all the files in the same folder,
|
||||
// and whether or not the library is in ld's searchpath.
|
||||
//
|
||||
// Instead we just make the requirement that renderdoc.json will always contain an absolute path
|
||||
// to the matching librenderdoc.so, so that we can check if it points to this build or another
|
||||
// build etc.
|
||||
//
|
||||
// Note there are three places to register layers - /usr, /etc and /home. The first is reserved
|
||||
// for distribution packages, so if it conflicts or needs to be deleted for this install to run,
|
||||
// we can't do that and have to just prompt the user. /etc we can mess with since that's for
|
||||
// non-distribution packages, but it will need root permissions.
|
||||
|
||||
bool exist[COUNT];
|
||||
bool match[COUNT];
|
||||
|
||||
int numExist = 0;
|
||||
int numMatch = 0;
|
||||
|
||||
for(int i = 0; i < COUNT; i++)
|
||||
{
|
||||
exist[i] = FileExists(layerRegistrationPath[i]);
|
||||
match[i] = (GetSOFromJSON(layerRegistrationPath[i]) == librenderdoc_path);
|
||||
|
||||
if(exist[i])
|
||||
numExist++;
|
||||
|
||||
if(match[i])
|
||||
numMatch++;
|
||||
}
|
||||
|
||||
flags = eVulkan_CouldElevate | eVulkan_UpdateAllowed;
|
||||
|
||||
if(numMatch >= 1)
|
||||
flags |= eVulkan_ThisInstallRegistered;
|
||||
|
||||
// if we only have one registration, check that it points to us. If so, we're good
|
||||
if(numExist == 1 && numMatch == 1)
|
||||
return false;
|
||||
|
||||
if(exist[USR] && !match[USR])
|
||||
otherJSONs.push_back(layerRegistrationPath[USR]);
|
||||
|
||||
if(exist[ETC] && !match[ETC])
|
||||
otherJSONs.push_back(layerRegistrationPath[ETC]);
|
||||
|
||||
if(exist[HOME] && !match[HOME])
|
||||
otherJSONs.push_back(layerRegistrationPath[HOME]);
|
||||
|
||||
if(!otherJSONs.empty())
|
||||
flags |= eVulkan_OtherInstallsRegistered;
|
||||
|
||||
if(exist[USR] && match[USR])
|
||||
{
|
||||
// just need to unregister others
|
||||
}
|
||||
else
|
||||
{
|
||||
myJSONs.push_back(layerRegistrationPath[ETC]);
|
||||
myJSONs.push_back(layerRegistrationPath[HOME]);
|
||||
}
|
||||
|
||||
if(exist[USR] && !match[USR])
|
||||
{
|
||||
flags = eVulkan_Unfixable | eVulkan_OtherInstallsRegistered;
|
||||
otherJSONs.clear();
|
||||
otherJSONs.push_back(layerRegistrationPath[USR]);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void VulkanReplay::InstallVulkanLayer(bool systemLevel)
|
||||
{
|
||||
// if we want to install to the system and there's a registration in $HOME, delete it
|
||||
if(systemLevel && FileExists(layerRegistrationPath[HOME]))
|
||||
{
|
||||
if(unlink(layerRegistrationPath[HOME].c_str()) < 0)
|
||||
{
|
||||
const char *const errtext = strerror(errno);
|
||||
RDCERR("Error removing %s: %s", layerRegistrationPath[HOME].c_str(), errtext);
|
||||
}
|
||||
}
|
||||
|
||||
// and vice-versa
|
||||
if(!systemLevel && FileExists(layerRegistrationPath[ETC]))
|
||||
{
|
||||
if(unlink(layerRegistrationPath[ETC].c_str()) < 0)
|
||||
{
|
||||
const char *const errtext = strerror(errno);
|
||||
RDCERR("Error removing %s: %s", layerRegistrationPath[ETC].c_str(), errtext);
|
||||
}
|
||||
}
|
||||
|
||||
int idx = systemLevel ? ETC : HOME;
|
||||
|
||||
string path = GetSOFromJSON(layerRegistrationPath[idx]);
|
||||
string libPath = GetThisLibPath();
|
||||
|
||||
if(path != libPath)
|
||||
{
|
||||
MakeParentDirs(layerRegistrationPath[idx]);
|
||||
|
||||
FILE *f = fopen(layerRegistrationPath[idx].c_str(), "w");
|
||||
|
||||
if(f)
|
||||
{
|
||||
fputs(GenerateJSON(libPath).c_str(), f);
|
||||
|
||||
fclose(f);
|
||||
}
|
||||
else
|
||||
{
|
||||
const char *const errtext = strerror(errno);
|
||||
RDCERR("Error writing %s: %s", layerRegistrationPath[idx].c_str(), errtext);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5630,4 +5630,14 @@ ReplayCreateStatus Vulkan_CreateReplayDevice(const char *logfile, IReplayDriver
|
||||
return eReplayCreate_Success;
|
||||
}
|
||||
|
||||
static DriverRegistration VkDriverRegistration(RDC_Vulkan, "Vulkan", &Vulkan_CreateReplayDevice);
|
||||
struct VulkanDriverRegistration
|
||||
{
|
||||
VulkanDriverRegistration()
|
||||
{
|
||||
RenderDoc::Inst().RegisterReplayProvider(RDC_Vulkan, "Vulkan", &Vulkan_CreateReplayDevice);
|
||||
RenderDoc::Inst().SetVulkanLayerCheck(&VulkanReplay::CheckVulkanLayer);
|
||||
RenderDoc::Inst().SetVulkanLayerInstall(&VulkanReplay::InstallVulkanLayer);
|
||||
}
|
||||
};
|
||||
|
||||
static VulkanDriverRegistration VkDriverRegistration;
|
||||
|
||||
@@ -258,6 +258,14 @@ public:
|
||||
// called before the VkDevice is destroyed, to shutdown any counters
|
||||
void PreDeviceShutdownCounters();
|
||||
|
||||
// used for vulkan layer bookkeeping. Ideally this should all be handled by installers/packages,
|
||||
// but for developers running builds locally or just in case, we need to be able to update the
|
||||
// layer registration ourselves.
|
||||
// These functions are defined in vk_<platform>.cpp
|
||||
static bool CheckVulkanLayer(uint32_t &flags, std::vector<std::string> &myJSONs,
|
||||
std::vector<std::string> &otherJSONs);
|
||||
static void InstallVulkanLayer(bool systemLevel);
|
||||
|
||||
private:
|
||||
struct OutputWindow
|
||||
{
|
||||
|
||||
@@ -157,3 +157,193 @@ VkBool32 WrappedVulkan::vkGetPhysicalDeviceWin32PresentationSupportKHR(VkPhysica
|
||||
}
|
||||
|
||||
const char *VulkanLibraryName = "vulkan-1.dll";
|
||||
|
||||
std::wstring GetJSONPath(bool wow6432)
|
||||
{
|
||||
wchar_t curFile[1024];
|
||||
GetModuleFileNameW(NULL, curFile, 1024);
|
||||
|
||||
wchar_t *lastSlash = wcsrchr(curFile, '\\');
|
||||
if(lastSlash)
|
||||
*(lastSlash + 1) = 0;
|
||||
|
||||
if(wow6432)
|
||||
wcscat_s(curFile, L"x86\\");
|
||||
|
||||
wcscat_s(curFile, L"renderdoc.json");
|
||||
|
||||
return curFile;
|
||||
}
|
||||
|
||||
static HKEY GetImplicitLayersKey(bool writeable, bool wow6432)
|
||||
{
|
||||
std::string basepath = "SOFTWARE\\";
|
||||
|
||||
if(wow6432)
|
||||
basepath += "Wow6432Node\\";
|
||||
|
||||
basepath += "Khronos\\Vulkan\\ImplicitLayers";
|
||||
|
||||
HKEY key = NULL;
|
||||
LSTATUS ret = ERROR_SUCCESS;
|
||||
|
||||
if(writeable)
|
||||
ret = RegCreateKeyExA(HKEY_LOCAL_MACHINE, basepath.c_str(), 0, NULL, 0, KEY_READ | KEY_WRITE,
|
||||
NULL, &key, NULL);
|
||||
else
|
||||
ret = RegOpenKeyExA(HKEY_LOCAL_MACHINE, basepath.c_str(), 0, KEY_READ, &key);
|
||||
|
||||
if(ret != ERROR_SUCCESS)
|
||||
{
|
||||
if(key)
|
||||
RegCloseKey(key);
|
||||
|
||||
// find to fail to open for read, the key may not exist
|
||||
if(writeable)
|
||||
RDCERR("Couldn't open %s for write", basepath.c_str());
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return key;
|
||||
}
|
||||
|
||||
bool ProcessImplicitLayersKey(HKEY key, const std::wstring &path,
|
||||
std::vector<std::string> *otherJSONs, bool deleteOthers)
|
||||
{
|
||||
bool thisRegistered = false;
|
||||
|
||||
wchar_t name[1024] = {};
|
||||
DWORD nameSize = 1024;
|
||||
DWORD idx = 0;
|
||||
|
||||
LONG ret = RegEnumValueW(key, idx++, name, &nameSize, NULL, NULL, NULL, NULL);
|
||||
|
||||
std::wstring myJSON = path;
|
||||
for(size_t i = 0; i < myJSON.size(); i++)
|
||||
myJSON[i] = towlower(myJSON[i]);
|
||||
|
||||
while(ret == ERROR_SUCCESS)
|
||||
{
|
||||
// convert the name here so we preserve casing
|
||||
std::string utf8name = StringFormat::Wide2UTF8(name);
|
||||
|
||||
for(DWORD i = 0; i <= nameSize && name[i]; i++)
|
||||
name[i] = towlower(name[i]);
|
||||
|
||||
if(wcscmp(name, myJSON.c_str()) == 0)
|
||||
{
|
||||
thisRegistered = true;
|
||||
}
|
||||
else if(wcsstr(name, L"renderdoc.json") != NULL)
|
||||
{
|
||||
if(otherJSONs)
|
||||
otherJSONs->push_back(utf8name);
|
||||
|
||||
if(deleteOthers)
|
||||
RegDeleteValueW(key, name);
|
||||
}
|
||||
|
||||
nameSize = 1024;
|
||||
ret = RegEnumValueW(key, idx++, name, &nameSize, NULL, NULL, NULL, NULL);
|
||||
}
|
||||
|
||||
return thisRegistered;
|
||||
}
|
||||
|
||||
bool VulkanReplay::CheckVulkanLayer(uint32_t &flags, std::vector<std::string> &myJSONs,
|
||||
std::vector<std::string> &otherJSONs)
|
||||
{
|
||||
std::wstring normalPath = GetJSONPath(false);
|
||||
myJSONs.push_back(StringFormat::Wide2UTF8(normalPath));
|
||||
|
||||
#if ENABLED(RDOC_X64)
|
||||
std::wstring wow6432Path = GetJSONPath(true);
|
||||
myJSONs.push_back(StringFormat::Wide2UTF8(wow6432Path));
|
||||
#endif
|
||||
|
||||
HKEY key = GetImplicitLayersKey(false, false);
|
||||
|
||||
// if we couldn't even get the ImplicitLayers reg key the system doesn't have the
|
||||
// vulkan runtime, so we return as if we are not registered (as that's the case).
|
||||
// People not using vulkan can either ignore the message, or click to set it up
|
||||
// and it will go away as we'll have rights to create it.
|
||||
if(!key)
|
||||
{
|
||||
flags = eVulkan_NeedElevation | eVulkan_RegisterAll;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool thisRegistered = ProcessImplicitLayersKey(key, normalPath, &otherJSONs, false);
|
||||
|
||||
RegCloseKey(key);
|
||||
|
||||
#if ENABLED(RDOC_X64)
|
||||
{
|
||||
key = GetImplicitLayersKey(false, true);
|
||||
|
||||
// if we're on 64-bit, the layer isn't registered unless both keys are registered.
|
||||
thisRegistered = false;
|
||||
|
||||
if(key)
|
||||
{
|
||||
thisRegistered = ProcessImplicitLayersKey(key, wow6432Path, &otherJSONs, false);
|
||||
|
||||
RegCloseKey(key);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
flags = eVulkan_NeedElevation | eVulkan_RegisterAll;
|
||||
|
||||
if(thisRegistered)
|
||||
flags |= eVulkan_ThisInstallRegistered;
|
||||
|
||||
if(!otherJSONs.empty())
|
||||
flags |= eVulkan_OtherInstallsRegistered;
|
||||
|
||||
// return true if any changes are needed
|
||||
return !otherJSONs.empty() || !thisRegistered;
|
||||
}
|
||||
|
||||
void VulkanReplay::InstallVulkanLayer(bool systemLevel)
|
||||
{
|
||||
HKEY key = GetImplicitLayersKey(true, false);
|
||||
|
||||
const DWORD zero = 0;
|
||||
|
||||
if(key)
|
||||
{
|
||||
std::wstring path = GetJSONPath(false);
|
||||
|
||||
// this function will delete all non-matching renderdoc.json values, and return true if our own
|
||||
// is registered
|
||||
bool thisRegistered = ProcessImplicitLayersKey(key, path, NULL, true);
|
||||
|
||||
if(!thisRegistered)
|
||||
RegSetValueExW(key, path.c_str(), 0, REG_DWORD, (const BYTE *)&zero, sizeof(zero));
|
||||
|
||||
RegCloseKey(key);
|
||||
}
|
||||
|
||||
// if we're a 64-bit process, update the 32-bit key
|
||||
#if ENABLED(RDOC_X64)
|
||||
{
|
||||
HKEY key = GetImplicitLayersKey(true, true);
|
||||
|
||||
if(key)
|
||||
{
|
||||
std::wstring path = GetJSONPath(true);
|
||||
|
||||
// this function will delete all non-matching renderdoc.json values, and return true if our
|
||||
// own is registered
|
||||
bool thisRegistered = ProcessImplicitLayersKey(key, path, NULL, true);
|
||||
|
||||
if(!thisRegistered)
|
||||
RegSetValueExW(key, path.c_str(), 0, REG_DWORD, (const BYTE *)&zero, sizeof(zero));
|
||||
|
||||
RegCloseKey(key);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -799,15 +799,37 @@ extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StartAndroidRemoteServer()
|
||||
"shell am start -n org.renderdoc.renderdoccmd/.Loader -e renderdoccmd remoteserver");
|
||||
}
|
||||
|
||||
extern "C" RENDERDOC_API bool RENDERDOC_CC RENDERDOC_NeedVulkanLayerRegistration(
|
||||
uint32_t *flags, rdctype::array<rdctype::str> *myJSONs, rdctype::array<rdctype::str> *otherJSONs)
|
||||
extern "C" RENDERDOC_API bool RENDERDOC_CC
|
||||
RENDERDOC_NeedVulkanLayerRegistration(uint32_t *flagsPtr, rdctype::array<rdctype::str> *myJSONsPtr,
|
||||
rdctype::array<rdctype::str> *otherJSONsPtr)
|
||||
{
|
||||
// stub
|
||||
uint32_t flags = 0;
|
||||
std::vector<std::string> myJSONs;
|
||||
std::vector<std::string> otherJSONs;
|
||||
|
||||
return false;
|
||||
bool ret = RenderDoc::Inst().NeedVulkanLayerRegistration(flags, myJSONs, otherJSONs);
|
||||
|
||||
if(flagsPtr)
|
||||
*flagsPtr = flags;
|
||||
|
||||
if(myJSONsPtr)
|
||||
{
|
||||
create_array(*myJSONsPtr, myJSONs.size());
|
||||
for(size_t i = 0; i < myJSONs.size(); i++)
|
||||
(*myJSONsPtr)[i] = myJSONs[i];
|
||||
}
|
||||
|
||||
if(otherJSONsPtr)
|
||||
{
|
||||
create_array(*otherJSONsPtr, otherJSONs.size());
|
||||
for(size_t i = 0; i < otherJSONs.size(); i++)
|
||||
(*otherJSONsPtr)[i] = otherJSONs[i];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_UpdateVulkanLayerRegistration(bool elevate)
|
||||
extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_UpdateVulkanLayerRegistration(bool systemLevel)
|
||||
{
|
||||
// stub
|
||||
RenderDoc::Inst().UpdateVulkanLayerRegistration(systemLevel);
|
||||
}
|
||||
@@ -51,124 +51,15 @@ void Daemonise()
|
||||
daemon(1, 0);
|
||||
}
|
||||
|
||||
// this is exported from vk_linux.cpp
|
||||
|
||||
#if defined(RENDERDOC_SUPPORT_VULKAN)
|
||||
|
||||
extern "C" void RENDERDOC_GetLayerJSON(char **txt, int *len);
|
||||
|
||||
#else
|
||||
|
||||
// just for ease of compiling, define a dummy function
|
||||
|
||||
void RENDERDOC_GetLayerJSON(char **txt, int *len)
|
||||
{
|
||||
static char dummy[] = "";
|
||||
*txt = dummy;
|
||||
*len = 0;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
static string GenerateJSON(const string &sopath)
|
||||
{
|
||||
char *txt = NULL;
|
||||
int len = 0;
|
||||
|
||||
RENDERDOC_GetLayerJSON(&txt, &len);
|
||||
|
||||
if(len <= 0)
|
||||
return "";
|
||||
|
||||
string json = string(txt, txt + len);
|
||||
|
||||
const char dllPathString[] = ".\\\\renderdoc.dll";
|
||||
|
||||
size_t idx = json.find(dllPathString);
|
||||
|
||||
return json.substr(0, idx) + sopath + json.substr(idx + sizeof(dllPathString) - 1);
|
||||
}
|
||||
|
||||
static bool FileExists(const string &path)
|
||||
{
|
||||
FILE *f = fopen(path.c_str(), "r");
|
||||
|
||||
if(f)
|
||||
{
|
||||
fclose(f);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static string GetSOFromJSON(const string &json)
|
||||
{
|
||||
char *json_string = new char[1024];
|
||||
memset(json_string, 0, 1024);
|
||||
|
||||
FILE *f = fopen(json.c_str(), "r");
|
||||
|
||||
if(f)
|
||||
{
|
||||
fread(json_string, 1, 1024, f);
|
||||
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
string ret = "";
|
||||
|
||||
// The line is:
|
||||
// "library_path": "/foo/bar/librenderdoc.so",
|
||||
char *c = strstr(json_string, "library_path");
|
||||
|
||||
if(c)
|
||||
{
|
||||
c += sizeof("library_path\": \"") - 1;
|
||||
|
||||
char *quote = strchr(c, '"');
|
||||
|
||||
if(quote)
|
||||
{
|
||||
*quote = 0;
|
||||
ret = c;
|
||||
}
|
||||
}
|
||||
|
||||
delete[] json_string;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
enum
|
||||
{
|
||||
USR,
|
||||
ETC,
|
||||
HOME,
|
||||
COUNT
|
||||
};
|
||||
|
||||
string layerRegistrationPath[COUNT] = {
|
||||
"/usr/share/vulkan/implicit_layer.d/renderdoc_capture.json",
|
||||
"/etc/vulkan/implicit_layer.d/renderdoc_capture.json",
|
||||
string(getenv("HOME")) + "/.local/share/vulkan/implicit_layer.d/renderdoc_capture.json"};
|
||||
|
||||
struct VulkanRegisterCommand : public Command
|
||||
{
|
||||
VulkanRegisterCommand(bool layer_exists[COUNT], const string &path)
|
||||
{
|
||||
etcExists = layer_exists[ETC];
|
||||
homeExists = layer_exists[HOME];
|
||||
libPath = path;
|
||||
}
|
||||
|
||||
VulkanRegisterCommand() {}
|
||||
virtual void AddOptions(cmdline::parser &parser)
|
||||
{
|
||||
parser.add("ignore", 'i', "Do nothing and don't warn about Vulkan layer issues.");
|
||||
parser.add(
|
||||
"system", '\0',
|
||||
"Install layer registration to /etc instead of $HOME/.local (requires root privileges)");
|
||||
parser.add("dry-run", 'n', "Don't perform any actions, instead print what would happen.");
|
||||
}
|
||||
virtual const char *Description()
|
||||
{
|
||||
@@ -208,233 +99,24 @@ struct VulkanRegisterCommand : public Command
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool system = (parser.exist("system"));
|
||||
bool dryrun = (parser.exist("dry-run"));
|
||||
|
||||
// if we want to install to the system and there's a registration in $HOME, delete it
|
||||
if(system && homeExists)
|
||||
{
|
||||
std::cout << "Removing '" << layerRegistrationPath[HOME] << "'" << std::endl;
|
||||
|
||||
if(!dryrun)
|
||||
{
|
||||
int ret = unlink(layerRegistrationPath[HOME].c_str());
|
||||
|
||||
if(ret < 0)
|
||||
{
|
||||
const char *const errtext = strerror(errno);
|
||||
std::cout << "Error - " << errtext << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// and vice-versa
|
||||
if(!system && etcExists)
|
||||
{
|
||||
std::cout << "Removing '" << layerRegistrationPath[ETC] << "'" << std::endl;
|
||||
|
||||
if(!dryrun)
|
||||
{
|
||||
int ret = unlink(layerRegistrationPath[ETC].c_str());
|
||||
|
||||
if(ret < 0)
|
||||
{
|
||||
const char *const errtext = strerror(errno);
|
||||
std::cout << "Error - " << errtext << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int idx = system ? ETC : HOME;
|
||||
|
||||
string path = GetSOFromJSON(layerRegistrationPath[idx]);
|
||||
|
||||
if(path != libPath)
|
||||
{
|
||||
if((system && !etcExists) || (!system && !homeExists))
|
||||
{
|
||||
std::cout << "Registering '" << layerRegistrationPath[idx] << "'" << std::endl;
|
||||
}
|
||||
else
|
||||
{
|
||||
std::cout << "Updating '" << layerRegistrationPath[idx] << "'" << std::endl;
|
||||
if(path == "")
|
||||
{
|
||||
std::cout << " JSON is corrupt or unrecognised, replacing with valid JSON pointing"
|
||||
<< std::endl;
|
||||
std::cout << " to '" << libPath << "'" << std::endl;
|
||||
}
|
||||
else
|
||||
{
|
||||
std::cout << " Repointing from '" << path << "'" << std::endl;
|
||||
std::cout << " to '" << libPath << "'" << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
if(!dryrun)
|
||||
{
|
||||
FILE *f = fopen(layerRegistrationPath[idx].c_str(), "w");
|
||||
|
||||
if(f)
|
||||
{
|
||||
fputs(GenerateJSON(libPath).c_str(), f);
|
||||
|
||||
fclose(f);
|
||||
}
|
||||
else
|
||||
{
|
||||
const char *const errtext = strerror(errno);
|
||||
std::cout << "Error - " << errtext << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
RENDERDOC_UpdateVulkanLayerRegistration(parser.exist("system"));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool etcExists;
|
||||
bool homeExists;
|
||||
string libPath;
|
||||
};
|
||||
|
||||
void VerifyVulkanLayer(int argc, char *argv[])
|
||||
{
|
||||
// see if the user has suppressed all this checking as a "I know what I'm doing" measure
|
||||
uint32_t flags = 0;
|
||||
rdctype::array<rdctype::str> myJSONs;
|
||||
rdctype::array<rdctype::str> otherJSONs;
|
||||
|
||||
string ignorePath = string(getenv("HOME")) + "/.renderdoc/ignore_vulkan_layer_issues";
|
||||
if(FileExists(ignorePath))
|
||||
return;
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////
|
||||
// check that there's only one layer registered, and it points to the same .so file that
|
||||
// we are running with in this instance of renderdoccmd
|
||||
|
||||
// this is a hack, but the only reliable way to find the absolute path to the library.
|
||||
// dladdr would be fine but it returns the wrong result for symbols in the library
|
||||
string librenderdoc_path;
|
||||
bool needUpdate = RENDERDOC_NeedVulkanLayerRegistration(&flags, &myJSONs, &otherJSONs);
|
||||
|
||||
if(!needUpdate)
|
||||
{
|
||||
FILE *f = fopen("/proc/self/maps", "r");
|
||||
|
||||
if(f)
|
||||
{
|
||||
// read the whole thing in one go. There's no need to try and be tight with
|
||||
// this allocation, so just make sure we can read everything.
|
||||
char *map_string = new char[1024 * 1024];
|
||||
memset(map_string, 0, 1024 * 1024);
|
||||
|
||||
fread(map_string, 1, 1024 * 1024, f);
|
||||
|
||||
fclose(f);
|
||||
|
||||
char *c = strstr(map_string, "/librenderdoc.so");
|
||||
|
||||
if(c)
|
||||
{
|
||||
// walk backwards until we hit the start of the line
|
||||
while(c > map_string)
|
||||
{
|
||||
c--;
|
||||
|
||||
if(c[0] == '\n')
|
||||
{
|
||||
c++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// walk forwards across the address range (00400000-0040c000)
|
||||
while(isalnum(c[0]) || c[0] == '-')
|
||||
c++;
|
||||
|
||||
// whitespace
|
||||
while(c[0] == ' ')
|
||||
c++;
|
||||
|
||||
// permissions (r-xp)
|
||||
while(isalpha(c[0]) || c[0] == '-')
|
||||
c++;
|
||||
|
||||
// whitespace
|
||||
while(c[0] == ' ')
|
||||
c++;
|
||||
|
||||
// offset (0000b000)
|
||||
while(isalnum(c[0]) || c[0] == '-')
|
||||
c++;
|
||||
|
||||
// whitespace
|
||||
while(c[0] == ' ')
|
||||
c++;
|
||||
|
||||
// dev
|
||||
while(isdigit(c[0]) || c[0] == ':')
|
||||
c++;
|
||||
|
||||
// whitespace
|
||||
while(c[0] == ' ')
|
||||
c++;
|
||||
|
||||
// inode
|
||||
while(isdigit(c[0]))
|
||||
c++;
|
||||
|
||||
// whitespace
|
||||
while(c[0] == ' ')
|
||||
c++;
|
||||
|
||||
// FINALLY we are at the start of the actual path
|
||||
char *end = strchr(c, '\n');
|
||||
|
||||
if(end)
|
||||
{
|
||||
librenderdoc_path = string(c, end - c);
|
||||
}
|
||||
}
|
||||
|
||||
delete[] map_string;
|
||||
}
|
||||
}
|
||||
|
||||
// it's impractical to determine whether the currently running RenderDoc build is just a loose
|
||||
// extract of a tarball or a distribution that decided to put all the files in the same folder,
|
||||
// and whether or not the library is in ld's searchpath.
|
||||
//
|
||||
// Instead we just make the requirement that renderdoc.json will always contain an absolute path
|
||||
// to the matching librenderdoc.so, so that we can check if it points to this build or another
|
||||
// build etc.
|
||||
//
|
||||
// Note there are three places to register layers - /usr, /etc and /home. The first is reserved
|
||||
// for distribution packages, so if it conflicts or needs to be deleted for this install to run,
|
||||
// we can't do that and have to just prompt the user. /etc we can mess with since that's for
|
||||
// non-distribution packages, but it will need root permissions.
|
||||
|
||||
bool exist[COUNT];
|
||||
bool match[COUNT];
|
||||
|
||||
int numExist = 0;
|
||||
int numMatch = 0;
|
||||
|
||||
for(int i = 0; i < COUNT; i++)
|
||||
{
|
||||
exist[i] = FileExists(layerRegistrationPath[i]);
|
||||
match[i] = (GetSOFromJSON(layerRegistrationPath[i]) == librenderdoc_path);
|
||||
|
||||
if(exist[i])
|
||||
numExist++;
|
||||
|
||||
if(match[i])
|
||||
numMatch++;
|
||||
}
|
||||
|
||||
// if we only have one registration, check that it points to us. If so, we're good
|
||||
if(numExist == 1 && numMatch == 1)
|
||||
return;
|
||||
|
||||
// if we're about to execute the command, don't print all this explanatory text.
|
||||
if(argc > 1 && !strcmp(argv[1], "vulkanregister"))
|
||||
{
|
||||
add_command("vulkanregister", new VulkanRegisterCommand(exist, librenderdoc_path));
|
||||
if(!(flags & eVulkan_Unfixable))
|
||||
add_command("vulkanregister", new VulkanRegisterCommand());
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -443,60 +125,40 @@ void VerifyVulkanLayer(int argc, char *argv[])
|
||||
std::cerr << "** Warning: Vulkan capture possibly not configured. **"
|
||||
<< std::endl;
|
||||
std::cerr << std::endl;
|
||||
if(numExist > 1)
|
||||
|
||||
if(flags & eVulkan_OtherInstallsRegistered)
|
||||
std::cerr << "Multiple RenderDoc layers are registered, possibly from different builds."
|
||||
<< std::endl;
|
||||
else if(numExist < 0)
|
||||
std::cerr << "RenderDoc layer is not registered." << std::endl;
|
||||
else
|
||||
std::cerr << "RenderDoc layer is registered, but to a different library." << std::endl;
|
||||
|
||||
if(!(flags & eVulkan_ThisInstallRegistered))
|
||||
std::cerr << "This build's RenderDoc layer is not registered." << std::endl;
|
||||
|
||||
std::cerr << "To fix this, the following actions must take place: " << std::endl << std::endl;
|
||||
|
||||
bool printed = false;
|
||||
const bool registerAll = (flags & eVulkan_RegisterAll);
|
||||
const bool updateAllowed = (flags & eVulkan_UpdateAllowed);
|
||||
|
||||
if(exist[USR] && !match[USR])
|
||||
{
|
||||
std::cerr << "* Unregister: '" << layerRegistrationPath[USR] << "'" << std::endl;
|
||||
printed = true;
|
||||
}
|
||||
for(const rdctype::str &j : otherJSONs)
|
||||
std::cerr << (updateAllowed ? "Unregister/update: " : "Unregister: ") << j.c_str() << std::endl;
|
||||
|
||||
if(exist[ETC] && !match[ETC])
|
||||
if(!(flags & eVulkan_ThisInstallRegistered))
|
||||
{
|
||||
std::cerr << "* Unregister or update: '" << layerRegistrationPath[ETC] << "'" << std::endl;
|
||||
printed = true;
|
||||
}
|
||||
|
||||
if(exist[HOME] && !match[HOME])
|
||||
{
|
||||
std::cerr << "* Unregister or update: '" << layerRegistrationPath[HOME] << "'" << std::endl;
|
||||
printed = true;
|
||||
}
|
||||
|
||||
if(printed)
|
||||
std::cerr << std::endl;
|
||||
|
||||
if(exist[USR] && match[USR])
|
||||
{
|
||||
// just need to unregister others
|
||||
}
|
||||
else
|
||||
{
|
||||
if(!exist[ETC] && !exist[HOME])
|
||||
if(registerAll)
|
||||
{
|
||||
std::cerr << "* Register either: '" << layerRegistrationPath[ETC] << "'" << std::endl;
|
||||
std::cerr << " or: '" << layerRegistrationPath[HOME] << "'" << std::endl;
|
||||
for(const rdctype::str &j : myJSONs)
|
||||
std::cerr << (updateAllowed ? "Register/update: " : "Register: ") << j.c_str() << std::endl;
|
||||
}
|
||||
else
|
||||
{
|
||||
std::cerr << "* Update or register either: '" << layerRegistrationPath[ETC] << "'" << std::endl;
|
||||
std::cerr << " or: '" << layerRegistrationPath[HOME] << "'"
|
||||
<< std::endl;
|
||||
std::cerr << (updateAllowed ? "Register one of:" : "Register/update one of:") << std::endl;
|
||||
for(const rdctype::str &j : myJSONs)
|
||||
std::cerr << " -- " << j.c_str() << "\n";
|
||||
}
|
||||
|
||||
std::cerr << std::endl;
|
||||
}
|
||||
|
||||
if(exist[USR] && !match[USR])
|
||||
std::cerr << std::endl;
|
||||
|
||||
if(flags & eVulkan_Unfixable)
|
||||
{
|
||||
std::cerr << "NOTE: The renderdoc layer registered in /usr is reserved for distribution"
|
||||
<< std::endl;
|
||||
@@ -535,7 +197,7 @@ void VerifyVulkanLayer(int argc, char *argv[])
|
||||
<< std::endl;
|
||||
std::cerr << std::endl;
|
||||
|
||||
add_command("vulkanregister", new VulkanRegisterCommand(exist, librenderdoc_path));
|
||||
add_command("vulkanregister", new VulkanRegisterCommand());
|
||||
}
|
||||
|
||||
void DisplayRendererPreview(ReplayRenderer *renderer, TextureDisplay &displayCfg, uint32_t width,
|
||||
|
||||
Reference in New Issue
Block a user