lee/renderdoc
1
0
Fork 0
mirror of https://github.com/baldurk/renderdoc.git synced 2026-09-16 10:45:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor env var handling, to handle layer as implicit

Browse Source
This commit is contained in:
baldurk
2016-02-07 18:49:57 +01:00
parent 1398b9fb56
commit 9ac8c365ee
8 changed files with 302 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files
renderdoc/core/core.cpp
+2
View File
@@ -195,6 +195,8 @@ void RenderDoc::Initialise()
if(!IsReplayApp())
{
Process::ApplyEnvironmentModification();
uint32_t port = RenderDoc_FirstCaptureNetworkPort;
Network::Socket *sock = Network::CreateServerSocket("0.0.0.0", port&0xffff, 4);
renderdoc/driver/vulkan/vk_linux.cpp
+1 -1
View File
@@ -164,4 +164,4 @@ VkResult WrappedVulkan::vkCreateXlibSurfaceKHR(
void *LoadVulkanLibrary()
{
return Process::LoadModule("libvulkan.so");
}
}
renderdoc/driver/vulkan/vk_replay.cpp
+1 -1
View File
@@ -4410,7 +4410,7 @@ void VulkanReplay::SetProxyBufferData(ResourceId bufid, byte *data, size_t dataS
VULKANNOTIMP("SetProxyTextureData");
}
// in vk_replay_platform.cpp
// in vk_<platform>.cpp
void *LoadVulkanLibrary();
ReplayCreateStatus Vulkan_CreateReplayDevice(const char *logfile, IReplayDriver **driver)
renderdoc/driver/vulkan/vk_tracelayer.cpp
+17
View File
@@ -35,9 +35,12 @@
#include "common/common.h"
#include "common/threading.h"
#include "serialise/string_utils.h"
#include "data/version.h"
#include "os/os_specific.h"
// this should be in the vulkan definition header
#ifdef WIN32
#undef VK_LAYER_EXPORT
@@ -47,6 +50,20 @@
void InitDeviceTable(const VkBaseLayerObject *obj);
void InitInstanceTable(const VkBaseLayerObject *obj);
// in vk_<platform>.cpp
bool LayerRegistered();
struct RegisterCallback
{
RegisterCallback()
{
// we assume the implicit layer is registered - the UI will prompt the user about installing it.
Process::RegisterEnvironmentModification(Process::EnvironmentModification(Process::eEnvModification_Replace, "ENABLE_VULKAN_RENDERDOC_CAPTURE", "1"));
}
};
static RegisterCallback registercb;
// RenderDoc State
// RenderDoc Intercepts, these must all be entry points with a dispatchable object
renderdoc/driver/vulkan/vk_win32.cpp
+1 -1
View File
@@ -106,4 +106,4 @@ VkBool32 WrappedVulkan::vkGetPhysicalDeviceWin32PresentationSupportKHR(
void *LoadVulkanLibrary()
{
return Process::LoadModule("vulkan-0.dll");
}
}
renderdoc/os/linux/linux_process.cpp
+117 -156
View File
@@ -35,7 +35,50 @@
#include "serialise/string_utils.h"
pid_t RunProcess(const char *app, const char *workingDir, const char *cmdLine, char *const *envp)
static vector<Process::EnvironmentModification> &GetEnvModifications()
{
static vector<Process::EnvironmentModification> envCallbacks;
return envCallbacks;
}
static map<string, string> EnvStringToEnvMap(const char **envstring)
{
map<string, string> ret;
const char **e = envstring;
while(*e)
{
const char *equals = strchr(*e, '=');
string name;
string value;
name.assign(e, equals);
value = equals+1;
ret[name] = value;
e++;
}
return ret;
}
void Process::RegisterEnvironmentModification(Process::EnvironmentModification modif)
{
GetEnvModifications().push_back(modif);
}
// on linux we apply environment changes before launching the program, as
// there is no support for injecting/loading renderdoc into a running program
// in any way, and we also have some environment changes that we *have* to make
// for correct hooking (LD_LIBRARY_PATH/LD_PRELOAD)
void Process::ApplyEnvironmentModification()
{
}
static pid_t RunProcess(const char *app, const char *workingDir, const char *cmdLine, char *const *envp)
{
if(!app) return (pid_t)0;
@@ -212,171 +255,89 @@ uint32_t Process::LaunchAndInjectIntoProcess(const char *app, const char *workin
RDCERR("Invalid empty 'app'");
return 0;
}
char **envp = NULL;
int nenv = 0;
for(; environ[nenv]; nenv++);
const int numEnvAdd = 7;
// LD_LIBRARY_PATH
// LD_PRELOAD
// VK_LAYER_PATH
// VK_DEVICE_LAYERS
// VK_INSTANCE_LAYERS
// RENDERDOC_CAPTUREOPTS
// RENDERDOC_LOGFILE
// might find these already existant in the environment
bool libpath = false;
bool preload = false;
bool layerdirs = false;
bool devicelayers = false;
bool instancelayers = false;
nenv += 1+numEnvAdd; // account for terminating NULL we need to replicate, and up to N additional environment varibales
envp = new char*[nenv];
string localpath;
FileIO::GetExecutableFilename(localpath);
localpath = dirname(localpath);
int i=0; int srci=0;
for(; i < nenv; srci++)
// turn environment string to a UTF-8 map
map<string, string> env = EnvStringToEnvMap(environ);
vector<EnvironmentModification> &modifications = GetEnvModifications();
string libpath;
{
if(environ[srci] == NULL)
{
envp[i] = NULL;
break;
}
size_t len = strlen(environ[srci])+1;
if(!strncmp(environ[srci], "LD_LIBRARY_PATH=", sizeof("LD_LIBRARY_PATH=")-1))
{
libpath = true;
envp[i] = new char[len+localpath.length()+1];
memcpy(envp[i], environ[srci], len);
strcat(envp[i], ":");
strcat(envp[i], localpath.c_str());
}
else if(!strncmp(environ[srci], "LD_PRELOAD=", sizeof("LD_PRELOAD=")-1))
{
preload = true;
envp[i] = new char[len+sizeof("librenderdoc.so")];
memcpy(envp[i], environ[srci], len);
strcat(envp[i], ":librenderdoc.so");
}
else if(!strncmp(environ[srci], "VK_LAYER_PATH=", sizeof("VK_LAYER_PATH=")-1))
{
layerdirs = true;
envp[i] = new char[len+1+localpath.length()+30];
memcpy(envp[i], environ[srci], len);
strcat(envp[i], ":");
strcat(envp[i], localpath.c_str());
}
else if(!strncmp(environ[srci], "VK_DEVICE_LAYERS=", sizeof("VK_DEVICE_LAYERS=")-1))
{
devicelayers = true;
envp[i] = new char[len+sizeof(":RenderDoc")];
memcpy(envp[i], environ[srci], len);
strcat(envp[i], ":RenderDoc");
}
else if(!strncmp(environ[srci], "VK_INSTANCE_LAYERS=", sizeof("VK_INSTANCE_LAYERS=")-1))
{
instancelayers = true;
envp[i] = new char[len+sizeof(":RenderDoc")];
memcpy(envp[i], environ[srci], len);
strcat(envp[i], ":RenderDoc");
}
else if(!strncmp(environ[srci], "RENDERDOC_", sizeof("RENDERDOC_")-1))
{
// skip this variable entirely
continue;
}
else
{
// basic copy
envp[i] = new char[len];
memcpy(envp[i], environ[srci], len);
}
i++;
FileIO::GetExecutableFilename(libpath);
libpath = dirname(libpath);
}
if(!libpath)
string optstr;
{
string e = StringFormat::Fmt("LD_LIBRARY_PATH=%s", localpath.c_str());
envp[i] = new char[e.length()+1];
memcpy(envp[i], e.c_str(), e.length()+1);
i++;
envp[i] = NULL;
}
if(!preload)
{
string e = StringFormat::Fmt("LD_PRELOAD=%s/librenderdoc.so", localpath.c_str());
envp[i] = new char[e.length()+1];
memcpy(envp[i], e.c_str(), e.length()+1);
i++;
envp[i] = NULL;
}
if(!layerdirs)
{
string e = StringFormat::Fmt("VK_LAYER_PATH=%s", localpath.c_str());
envp[i] = new char[e.length()+1];
memcpy(envp[i], e.c_str(), e.length()+1);
i++;
envp[i] = NULL;
}
if(!devicelayers)
{
string e = StringFormat::Fmt("VK_DEVICE_LAYERS=RenderDoc");
envp[i] = new char[e.length()+1];
memcpy(envp[i], e.c_str(), e.length()+1);
i++;
envp[i] = NULL;
}
if(!instancelayers)
{
string e = StringFormat::Fmt("VK_INSTANCE_LAYERS=RenderDoc");
envp[i] = new char[e.length()+1];
memcpy(envp[i], e.c_str(), e.length()+1);
i++;
envp[i] = NULL;
}
if(opts)
{
string optstr;
optstr.reserve(sizeof(CaptureOptions)*2+1);
byte *b = (byte *)opts;
for(size_t i=0; i < sizeof(CaptureOptions); i++)
{
optstr.reserve(sizeof(CaptureOptions)*2+1);
byte *b = (byte *)opts;
for(size_t i=0; i < sizeof(CaptureOptions); i++)
{
optstr.push_back(char( 'a' + ((b[i] >> 4)&0xf) ));
optstr.push_back(char( 'a' + ((b[i] )&0xf) ));
}
optstr.push_back(char( 'a' + ((b[i] >> 4)&0xf) ));
optstr.push_back(char( 'a' + ((b[i] )&0xf) ));
}
string e = StringFormat::Fmt("RENDERDOC_CAPTUREOPTS=%s", optstr.c_str());
envp[i] = new char[e.length()+1];
memcpy(envp[i], e.c_str(), e.length()+1);
i++;
envp[i] = NULL;
}
if(logfile)
modifications.push_back(EnvironmentModification(eEnvModification_AppendPlatform, "LD_LIBRARY_PATH", libpath.c_str()));
modifications.push_back(EnvironmentModification(eEnvModification_AppendPlatform, "LD_PRELOAD", "librenderdoc.so"));
modifications.push_back(EnvironmentModification(eEnvModification_Replace, "RENDERDOC_LOGFILE", logfile));
modifications.push_back(EnvironmentModification(eEnvModification_Replace, "RENDERDOC_CAPTUREOPTS", optstr.c_str()));
for(size_t i=0; i < modifications.size(); i++)
{
string e = StringFormat::Fmt("RENDERDOC_LOGFILE=%s", logfile);
envp[i] = new char[e.length()+1];
memcpy(envp[i], e.c_str(), e.length()+1);
EnvironmentModification &m = modifications[i];
string value = env[m.name];
switch(m.type)
{
case eEnvModification_Replace:
value = m.value;
break;
case eEnvModification_Append:
value += m.value;
break;
case eEnvModification_AppendColon:
if(!value.empty())
value += ":";
value += m.value;
break;
case eEnvModification_AppendPlatform:
case eEnvModification_AppendSemiColon:
if(!value.empty())
value += ";";
value += m.value;
break;
case eEnvModification_Prepend:
value = m.value + value;
break;
case eEnvModification_PrependColon:
if(!value.empty())
value = m.value + ":" + value;
else
value = m.value;
break;
case eEnvModification_PrependPlatform:
case eEnvModification_PrependSemiColon:
if(!value.empty())
value = m.value + ";" + value;
else
value = m.value;
break;
default:
RDCERR("Unexpected environment modification type");
}
}
char **envp = new char *[env.size()+1];
envp[env.size()] = NULL;
int i=0;
for(auto it=env.begin(); it != env.end(); it++)
{
string envline = it->first + "=" + it->second;
envp[i] = new char[envline.size()+1];
memcpy(envp[i], envline.c_str(), envline.size()+1);
i++;
envp[i] = NULL;
}
pid_t childPid = RunProcess(app, workingDir, cmdLine, envp);
renderdoc/os/os_specific.h
+29
View File
@@ -38,13 +38,42 @@
#include <string>
#include <vector>
#include <map>
using std::string;
using std::vector;
using std::map;
struct CaptureOptions;
namespace Process
{
enum ModificationType
{
eEnvModification_Replace = 0,
// prepend/append options will replace if there is no existing variable
eEnvModification_Append, // append with no separators
eEnvModification_AppendColon, // append, separated by colons
eEnvModification_AppendSemiColon, // append, separated by semi-colons
eEnvModification_AppendPlatform, // append, separated by colons for linux & semi-colons for windows
eEnvModification_Prepend, // prepend with no separators
eEnvModification_PrependColon, // prepend, separated by colons
eEnvModification_PrependSemiColon, // prepend, separated by semi-colons
eEnvModification_PrependPlatform, // prepend, separated by colons for linux & semi-colons for windows
};
struct EnvironmentModification
{
EnvironmentModification() : type(eEnvModification_Replace), name(""), value("") {}
EnvironmentModification(ModificationType t, const char *n, const char *v) : type(t), name(n), value(v) {}
ModificationType type;
string name;
string value;
};
void RegisterEnvironmentModification(EnvironmentModification modif);
void ApplyEnvironmentModification();
void StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions *opts);
uint32_t InjectIntoProcess(uint32_t pid, const char *logfile, const CaptureOptions *opts, bool waitForExit);
uint32_t LaunchProcess(const char *app, const char *workingDir, const char *cmdLine);
renderdoc/os/win32/win32_process.cpp
+134 -96
View File
@@ -36,6 +36,104 @@
#include <string>
using std::string;
static vector<Process::EnvironmentModification> &GetEnvModifications()
{
static vector<Process::EnvironmentModification> envCallbacks;
return envCallbacks;
}
static map<string, string> EnvStringToEnvMap(const wchar_t *envstring)
{
map<string, string> ret;
const wchar_t *e = envstring;
while(*e)
{
const wchar_t *equals = wcschr(e, L'=');
wstring name;
wstring value;
name.assign(e, equals);
value = equals+1;
ret[StringFormat::Wide2UTF8(name)] = StringFormat::Wide2UTF8(value);
e += name.size(); // jump to =
e++; // advance past it
e += value.size(); // jump to \0
e++; // advance past it
}
return ret;
}
void Process::RegisterEnvironmentModification(Process::EnvironmentModification modif)
{
GetEnvModifications().push_back(modif);
}
// on windows we apply environment changes here, after process initialisation
// but before any real work (in RenderDoc::Initialise) so that we support
// injecting the dll into processes we didn't launch (ie didn't control the
// starting environment for), or even the application loading the dll itself
// without any interaction with our replay app.
void Process::ApplyEnvironmentModification()
{
// turn environment string to a UTF-8 map
map<string, string> currentEnv = EnvStringToEnvMap(GetEnvironmentStringsW());
vector<EnvironmentModification> &modifications = GetEnvModifications();
for(size_t i=0; i < modifications.size(); i++)
{
EnvironmentModification &m = modifications[i];
string value = currentEnv[m.name];
switch(m.type)
{
case eEnvModification_Replace:
value = m.value;
break;
case eEnvModification_Append:
value += m.value;
break;
case eEnvModification_AppendColon:
if(!value.empty())
value += ":";
value += m.value;
break;
case eEnvModification_AppendPlatform:
case eEnvModification_AppendSemiColon:
if(!value.empty())
value += ";";
value += m.value;
break;
case eEnvModification_Prepend:
value = m.value + value;
break;
case eEnvModification_PrependColon:
if(!value.empty())
value = m.value + ":" + value;
else
value = m.value;
break;
case eEnvModification_PrependPlatform:
case eEnvModification_PrependSemiColon:
if(!value.empty())
value = m.value + ";" + value;
else
value = m.value;
break;
default:
RDCERR("Unexpected environment modification type");
}
SetEnvironmentVariableW(StringFormat::UTF82Wide(m.name).c_str(), StringFormat::UTF82Wide(value).c_str());
}
}
// helpers for various shims and dlls etc, not part of the public API
extern "C" __declspec(dllexport)
void __cdecl RENDERDOC_GetRemoteAccessIdent(uint32_t *ident)
@@ -54,7 +152,6 @@ void __cdecl RENDERDOC_SetLogFile(const char *log)
{
if(log) RenderDoc::Inst().SetLogFile(log);
}
void InjectDLL(HANDLE hProcess, wstring libName)
{
wchar_t dllPath[MAX_PATH + 1] = {0};
@@ -202,116 +299,58 @@ void InjectFunctionCall(HANDLE hProcess, uintptr_t renderdoc_remote, const char
static PROCESS_INFORMATION RunProcess(const char *app, const char *workingDir, const char *cmdLine)
{
PROCESS_INFORMATION pi;
STARTUPINFO si;
SECURITY_ATTRIBUTES pSec;
SECURITY_ATTRIBUTES tSec;
PROCESS_INFORMATION pi;
STARTUPINFO si;
SECURITY_ATTRIBUTES pSec;
SECURITY_ATTRIBUTES tSec;
RDCEraseEl(pi);
RDCEraseEl(si);
RDCEraseEl(pSec);
RDCEraseEl(tSec);
RDCEraseEl(pi);
RDCEraseEl(si);
RDCEraseEl(pSec);
RDCEraseEl(tSec);
pSec.nLength = sizeof(pSec);
tSec.nLength = sizeof(tSec);
pSec.nLength = sizeof(pSec);
tSec.nLength = sizeof(tSec);
wstring workdir = L"";
wstring workdir = L"";
if (workingDir != NULL && workingDir[0] != 0)
workdir = StringFormat::UTF82Wide(string(workingDir));
else
workdir = StringFormat::UTF82Wide(dirname(string(app)));
if(workingDir != NULL && workingDir[0] != 0)
workdir = StringFormat::UTF82Wide(string(workingDir));
else
workdir = StringFormat::UTF82Wide(dirname(string(app)));
wchar_t *paramsAlloc = NULL;
wchar_t *paramsAlloc = NULL;
wstring wapp = StringFormat::UTF82Wide(string(app));
wstring wapp = StringFormat::UTF82Wide(string(app));
// CreateProcessW can modify the params, need space.
size_t len = wapp.length() + 10;
// CreateProcessW can modify the params, need space.
size_t len = wapp.length()+10;
wstring wcmd = L"";
wstring wcmd = L"";
if (cmdLine != NULL && cmdLine[0] != 0)
{
wcmd = StringFormat::UTF82Wide(string(cmdLine));
len += wcmd.length();
}
if(cmdLine != NULL && cmdLine[0] != 0)
{
wcmd = StringFormat::UTF82Wide(string(cmdLine));
len += wcmd.length();
}
paramsAlloc = new wchar_t[len];
paramsAlloc = new wchar_t[len];
RDCEraseMem(paramsAlloc, len*sizeof(wchar_t));
RDCEraseMem(paramsAlloc, len*sizeof(wchar_t));
wcscpy_s(paramsAlloc, len, L"\"");
wcscat_s(paramsAlloc, len, wapp.c_str());
wcscat_s(paramsAlloc, len, L"\"");
wcscpy_s(paramsAlloc, len, L"\"");
wcscat_s(paramsAlloc, len, wapp.c_str());
wcscat_s(paramsAlloc, len, L"\"");
if (cmdLine != NULL && cmdLine[0] != 0)
{
wcscat_s(paramsAlloc, len, L" ");
wcscat_s(paramsAlloc, len, wcmd.c_str());
}
// VKTODOLOW refactor all this once loader supports implicit layers
const wchar_t *myEnv = GetEnvironmentStringsW();
wstring newEnv;
// copy up to the terminating "\0\0"
bool sawpath = false;
bool sawdevice = false;
bool sawinstance = false;
while (myEnv[0] != L'\0' || myEnv[1] != L'\0')
{
if (!wcsncmp(&myEnv[0], L"VK_LAYER_PATH=", sizeof("VK_LAYER_PATH=") - 1))
{
sawpath = true;
newEnv += L"VK_LAYER_PATH=";
newEnv += dirname(StringFormat::UTF82Wide(FileIO::GetReplayAppFilename()));
newEnv += L";";
myEnv += (sizeof("VK_LAYER_PATH=") - 1);
}
else if (!wcsncmp(&myEnv[0], L"VK_DEVICE_LAYERS=", sizeof("VK_DEVICE_LAYERS=") - 1))
{
sawdevice = true;
newEnv += L"VK_DEVICE_LAYERS=VK_LAYER_RENDERDOC_Capture;";
myEnv += (sizeof("VK_DEVICE_LAYERS=") - 1);
}
else if (!wcsncmp(&myEnv[0], L"VK_INSTANCE_LAYERS=", sizeof("VK_INSTANCE_LAYERS=") - 1))
{
sawinstance = true;
newEnv += L"VK_INSTANCE_LAYERS=VK_LAYER_RENDERDOC_Capture;";
myEnv += (sizeof("VK_INSTANCE_LAYERS=") - 1);
}
else
newEnv.push_back(*(myEnv++));
}
newEnv.push_back(L'\0');
if (!sawpath)
{
newEnv += L"VK_LAYER_PATH=";
newEnv += dirname(StringFormat::UTF82Wide(FileIO::GetReplayAppFilename()));
newEnv.push_back(L'\0');
}
if (!sawdevice)
{
newEnv += L"VK_DEVICE_LAYERS=VK_LAYER_RENDERDOC_Capture";
newEnv.push_back(L'\0');
}
if (!sawinstance)
{
newEnv += L"VK_INSTANCE_LAYERS=VK_LAYER_RENDERDOC_Capture";
newEnv.push_back(L'\0');
}
newEnv.push_back(L'\0');
if(cmdLine != NULL && cmdLine[0] != 0)
{
wcscat_s(paramsAlloc, len, L" ");
wcscat_s(paramsAlloc, len, wcmd.c_str());
}
BOOL retValue = CreateProcessW(NULL, paramsAlloc,
&pSec, &tSec, false, CREATE_SUSPENDED|CREATE_UNICODE_ENVIRONMENT,
(void *)newEnv.c_str(), workdir.c_str(), &si, &pi);
NULL, workdir.c_str(), &si, &pi);
SAFE_DELETE_ARRAY(paramsAlloc);
@@ -644,4 +683,3 @@ uint32_t Process::GetCurrentPID()
{
return (uint32_t)GetCurrentProcessId();
}