From 9f4a60028ed9d938fa8e60d334a2ba62f9c37966 Mon Sep 17 00:00:00 2001 From: baldurk Date: Wed, 28 Sep 2016 17:55:45 +0200 Subject: [PATCH] Handle injecting environment variable params * This also fixes the problem of capturing 32-bit programs with 64-bit RenderDoc failing to properly insert environment variables and error'ing when it tries to do it directly. --- renderdoc/api/replay/renderdoc_replay.h | 2 +- renderdoc/os/os_specific.h | 4 +- renderdoc/os/posix/posix_process.cpp | 4 +- renderdoc/os/win32/sys_win32_hooks.cpp | 4 +- renderdoc/os/win32/win32_process.cpp | 145 +++++++++++++------ renderdoc/replay/entry_points.cpp | 5 +- renderdoccmd/renderdoccmd.cpp | 88 ++++++++++- renderdocui/Interop/StaticExports.cs | 21 ++- renderdocui/Windows/Dialogs/CaptureDialog.cs | 4 +- renderdocui/Windows/MainWindow.cs | 4 +- 10 files changed, 214 insertions(+), 67 deletions(-) diff --git a/renderdoc/api/replay/renderdoc_replay.h b/renderdoc/api/replay/renderdoc_replay.h index 544f8742a..0c2e41755 100644 --- a/renderdoc/api/replay/renderdoc_replay.h +++ b/renderdoc/api/replay/renderdoc_replay.h @@ -637,7 +637,7 @@ extern "C" RENDERDOC_API uint32_t RENDERDOC_CC RENDERDOC_ExecuteAndInject(const char *app, const char *workingDir, const char *cmdLine, void *env, const char *logfile, const CaptureOptions *opts, bool32 waitForExit); extern "C" RENDERDOC_API uint32_t RENDERDOC_CC RENDERDOC_InjectIntoProcess( - uint32_t pid, const char *logfile, const CaptureOptions *opts, bool32 waitForExit); + uint32_t pid, void *env, const char *logfile, const CaptureOptions *opts, bool32 waitForExit); ////////////////////////////////////////////////////////////////////////// // Miscellaneous! diff --git a/renderdoc/os/os_specific.h b/renderdoc/os/os_specific.h index b3961241a..3a70be85e 100644 --- a/renderdoc/os/os_specific.h +++ b/renderdoc/os/os_specific.h @@ -80,8 +80,8 @@ void RegisterEnvironmentModification(EnvironmentModification modif); void ApplyEnvironmentModification(); void StartGlobalHook(const char *pathmatch, const char *logfile, const CaptureOptions *opts); -uint32_t InjectIntoProcess(uint32_t pid, const char *logfile, const CaptureOptions *opts, - bool waitForExit); +uint32_t InjectIntoProcess(uint32_t pid, EnvironmentModification *env, const char *logfile, + const CaptureOptions *opts, bool waitForExit); uint32_t LaunchProcess(const char *app, const char *workingDir, const char *cmdLine); uint32_t LaunchAndInjectIntoProcess(const char *app, const char *workingDir, const char *cmdLine, EnvironmentModification *env, const char *logfile, diff --git a/renderdoc/os/posix/posix_process.cpp b/renderdoc/os/posix/posix_process.cpp index 6ea776708..3931c826b 100644 --- a/renderdoc/os/posix/posix_process.cpp +++ b/renderdoc/os/posix/posix_process.cpp @@ -341,8 +341,8 @@ static pid_t RunProcess(const char *app, const char *workingDir, const char *cmd return childPid; } -uint32_t Process::InjectIntoProcess(uint32_t pid, const char *logfile, const CaptureOptions *opts, - bool waitForExit) +uint32_t Process::InjectIntoProcess(uint32_t pid, EnvironmentModification *env, const char *logfile, + const CaptureOptions *opts, bool waitForExit) { RDCUNIMPLEMENTED("Injecting into already running processes on linux"); return 0; diff --git a/renderdoc/os/win32/sys_win32_hooks.cpp b/renderdoc/os/win32/sys_win32_hooks.cpp index db2816425..f7d56ef80 100644 --- a/renderdoc/os/win32/sys_win32_hooks.cpp +++ b/renderdoc/os/win32/sys_win32_hooks.cpp @@ -146,7 +146,7 @@ private: if(inject) { // inherit logfile and capture options - uint32_t ident = RENDERDOC_InjectIntoProcess(lpProcessInformation->dwProcessId, + uint32_t ident = RENDERDOC_InjectIntoProcess(lpProcessInformation->dwProcessId, NULL, RenderDoc::Inst().GetLogFile(), &RenderDoc::Inst().GetCaptureOptions(), false); @@ -233,7 +233,7 @@ private: if(inject) { // inherit logfile and capture options - uint32_t ident = RENDERDOC_InjectIntoProcess(lpProcessInformation->dwProcessId, + uint32_t ident = RENDERDOC_InjectIntoProcess(lpProcessInformation->dwProcessId, NULL, RenderDoc::Inst().GetLogFile(), &RenderDoc::Inst().GetCaptureOptions(), false); diff --git a/renderdoc/os/win32/win32_process.cpp b/renderdoc/os/win32/win32_process.cpp index 48a5fc6ec..0fc72de1a 100644 --- a/renderdoc/os/win32/win32_process.cpp +++ b/renderdoc/os/win32/win32_process.cpp @@ -439,8 +439,8 @@ static PROCESS_INFORMATION RunProcess(const char *app, const char *workingDir, c return pi; } -uint32_t Process::InjectIntoProcess(uint32_t pid, const char *logfile, const CaptureOptions *opts, - bool waitForExit) +uint32_t Process::InjectIntoProcess(uint32_t pid, EnvironmentModification *env, const char *logfile, + const CaptureOptions *opts, bool waitForExit) { CaptureOptions options; if(opts) @@ -539,8 +539,6 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, const char *logfile, const Cap pSec.nLength = sizeof(pSec); tSec.nLength = sizeof(tSec); - wchar_t *paramsAlloc = new wchar_t[2048]; - // serialise to string with two chars per byte string optstr; { @@ -553,13 +551,78 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, const char *logfile, const Cap } } + wchar_t *paramsAlloc = new wchar_t[2048]; + _snwprintf_s(paramsAlloc, 2047, 2047, L"\"%ls\" cap32for64 --pid=%d --log=\"%ls\" --capopts=\"%hs\"", renderdocPath, pid, wlogfile.c_str(), optstr.c_str()); paramsAlloc[2047] = 0; - BOOL retValue = CreateProcessW(NULL, paramsAlloc, &pSec, &tSec, false, CREATE_SUSPENDED, NULL, + wchar_t *commandLine = paramsAlloc; + + wstring cmdWithEnv; + + if(env) + { + cmdWithEnv = paramsAlloc; + + for(;;) + { + string name = trim(env->name); + string value = env->value; + ModificationType type = env->type; + + if(name == "") + break; + + cmdWithEnv += L" +env-"; + switch(type) + { + case eEnvModification_Replace: cmdWithEnv += L"replace "; break; + + case eEnvModification_AppendPlatform: cmdWithEnv += L"append-platform "; break; + + case eEnvModification_AppendSemiColon: cmdWithEnv += L"append-semicolon "; break; + case eEnvModification_AppendColon: cmdWithEnv += L"append-colon "; break; + case eEnvModification_Append: cmdWithEnv += L"append "; break; + + case eEnvModification_PrependPlatform: cmdWithEnv += L"prepend-platform "; break; + + case eEnvModification_PrependSemiColon: cmdWithEnv += L"prepend-semicolon "; break; + case eEnvModification_PrependColon: cmdWithEnv += L"prepend-colon "; break; + case eEnvModification_Prepend: cmdWithEnv += L"prepend "; break; + } + + // escape the parameters + for(auto it = name.begin(); it != name.end(); ++it) + { + if(*it == '"') + it = name.insert(it, '\\') + 1; + } + + for(auto it = value.begin(); it != value.end(); ++it) + { + if(*it == '"') + it = value.insert(it, '\\') + 1; + } + + if(name.back() == '\\') + name += "\\"; + + if(value.back() == '\\') + value += "\\"; + + cmdWithEnv += L"\"" + StringFormat::UTF82Wide(name) + L"\" "; + cmdWithEnv += L"\"" + StringFormat::UTF82Wide(value) + L"\" "; + + env++; + } + + commandLine = (wchar_t *)cmdWithEnv.c_str(); + } + + BOOL retValue = CreateProcessW(NULL, commandLine, &pSec, &tSec, false, CREATE_SUSPENDED, NULL, NULL, &si, &pi); SAFE_DELETE_ARRAY(paramsAlloc); @@ -610,6 +673,31 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, const char *logfile, const Cap InjectFunctionCall(hProcess, loc, "RENDERDOC_GetTargetControlIdent", &controlident, sizeof(controlident)); + + if(env) + { + for(;;) + { + string name = trim(env->name); + string value = env->value; + ModificationType type = env->type; + + if(name == "") + break; + + InjectFunctionCall(hProcess, loc, "RENDERDOC_EnvModName", (void *)name.c_str(), + name.size() + 1); + InjectFunctionCall(hProcess, loc, "RENDERDOC_EnvModValue", (void *)value.c_str(), + value.size() + 1); + InjectFunctionCall(hProcess, loc, "RENDERDOC_EnvMod", &type, sizeof(type)); + + env++; + } + + // parameter is unused + InjectFunctionCall(hProcess, loc, "RENDERDOC_ApplyEnvMods", env, + sizeof(EnvironmentModification)); + } } if(waitForExit) @@ -657,55 +745,18 @@ uint32_t Process::LaunchAndInjectIntoProcess(const char *app, const char *workin if(pi.dwProcessId == 0) return 0; - uint32_t ret = InjectIntoProcess(pi.dwProcessId, logfile, opts, false); + uint32_t ret = InjectIntoProcess(pi.dwProcessId, env, logfile, opts, false); + + CloseHandle(pi.hProcess); + ResumeThread(pi.hThread); + ResumeThread(pi.hThread); if(ret == 0) { - ResumeThread(pi.hThread); CloseHandle(pi.hThread); - CloseHandle(pi.hProcess); return 0; } - if(env) - { - uintptr_t loc = FindRemoteDLL(pi.dwProcessId, L"renderdoc.dll"); - - if(loc == 0) - { - RDCERR("Can't locate renderdoc.dll in remote PID %d", pi.dwProcessId); - } - else - { - for(;;) - { - string name = trim(env->name); - string value = env->value; - ModificationType type = env->type; - - if(name == "") - break; - - InjectFunctionCall(pi.hProcess, loc, "RENDERDOC_EnvModName", (void *)name.c_str(), - name.size() + 1); - InjectFunctionCall(pi.hProcess, loc, "RENDERDOC_EnvModValue", (void *)value.c_str(), - value.size() + 1); - InjectFunctionCall(pi.hProcess, loc, "RENDERDOC_EnvMod", &type, sizeof(type)); - - env++; - } - - // parameter is unused - InjectFunctionCall(pi.hProcess, loc, "RENDERDOC_ApplyEnvMods", env, - sizeof(EnvironmentModification)); - } - } - - // done with it - CloseHandle(pi.hProcess); - - ResumeThread(pi.hThread); - if(waitForExit) WaitForSingleObject(pi.hThread, INFINITE); diff --git a/renderdoc/replay/entry_points.cpp b/renderdoc/replay/entry_points.cpp index 3e05ef55f..adce84f98 100644 --- a/renderdoc/replay/entry_points.cpp +++ b/renderdoc/replay/entry_points.cpp @@ -402,9 +402,10 @@ extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StartGlobalHook(const char } extern "C" RENDERDOC_API uint32_t RENDERDOC_CC RENDERDOC_InjectIntoProcess( - uint32_t pid, const char *logfile, const CaptureOptions *opts, bool32 waitForExit) + uint32_t pid, void *env, const char *logfile, const CaptureOptions *opts, bool32 waitForExit) { - return Process::InjectIntoProcess(pid, logfile, opts, waitForExit != 0); + return Process::InjectIntoProcess(pid, (Process::EnvironmentModification *)env, logfile, opts, + waitForExit != 0); } extern "C" RENDERDOC_API bool32 RENDERDOC_CC RENDERDOC_GetThumbnail(const char *filename, byte *buf, diff --git a/renderdoccmd/renderdoccmd.cpp b/renderdoccmd/renderdoccmd.cpp index 14c570b3d..9cfb358de 100644 --- a/renderdoccmd/renderdoccmd.cpp +++ b/renderdoccmd/renderdoccmd.cpp @@ -374,8 +374,8 @@ struct InjectCommand : public Command std::cout << "Injecting into PID " << PID << std::endl; - uint32_t ident = RENDERDOC_InjectIntoProcess(PID, logFile.empty() ? "" : logFile.c_str(), &opts, - parser.exist("wait-for-exit")); + uint32_t ident = RENDERDOC_InjectIntoProcess(PID, NULL, logFile.empty() ? "" : logFile.c_str(), + &opts, parser.exist("wait-for-exit")); if(ident == 0) { @@ -539,6 +539,7 @@ struct Cap32For64Command : public Command parser.add("pid", 0, ""); parser.add("log", 0, ""); parser.add("capopts", 0, ""); + parser.stop_at_rest(true); } virtual const char *Description() { return "Internal use only!"; } virtual bool IsInternalOnly() { return true; } @@ -548,8 +549,87 @@ struct Cap32For64Command : public Command CaptureOptions cmdopts; readCapOpts(parser.get("capopts").c_str(), &cmdopts); - return RENDERDOC_InjectIntoProcess(parser.get("pid"), - parser.get("log").c_str(), &cmdopts, false); + std::vector rest = parser.rest(); + + if(rest.size() % 3 != 0) + { + std::cerr << "Invalid generated cap32for64 command rest.size() == " << rest.size() << std::endl; + return 0; + } + + int numEnvs = int(rest.size() / 3); + + void *env = RENDERDOC_MakeEnvironmentModificationList(numEnvs); + + for(int i = 0; i < numEnvs; i++) + { + string typeString = rest[i * 3 + 0]; + + EnvironmentModificationType type = eEnvMod_Set; + EnvironmentSeparator sep = eEnvSep_None; + + if(typeString == "+env-replace") + { + type = eEnvMod_Set; + sep = eEnvSep_None; + } + else if(typeString == "+env-append-platform") + { + type = eEnvMod_Append; + sep = eEnvSep_Platform; + } + else if(typeString == "+env-append-semicolon") + { + type = eEnvMod_Append; + sep = eEnvSep_SemiColon; + } + else if(typeString == "+env-append-colon") + { + type = eEnvMod_Append; + sep = eEnvSep_Colon; + } + else if(typeString == "+env-append") + { + type = eEnvMod_Append; + sep = eEnvSep_None; + } + else if(typeString == "+env-prepend-platform") + { + type = eEnvMod_Prepend; + sep = eEnvSep_Platform; + } + else if(typeString == "+env-prepend-semicolon") + { + type = eEnvMod_Prepend; + sep = eEnvSep_SemiColon; + } + else if(typeString == "+env-prepend-colon") + { + type = eEnvMod_Prepend; + sep = eEnvSep_Colon; + } + else if(typeString == "+env-prepend") + { + type = eEnvMod_Prepend; + sep = eEnvSep_None; + } + else + { + std::cerr << "Invalid generated cap32for64 env '" << rest[i * 3 + 0] << std::endl; + RENDERDOC_FreeEnvironmentModificationList(env); + return 0; + } + + RENDERDOC_SetEnvironmentModification(env, i, rest[i * 3 + 1].c_str(), rest[i * 3 + 2].c_str(), + type, sep); + } + + int ret = RENDERDOC_InjectIntoProcess(parser.get("pid"), env, + parser.get("log").c_str(), &cmdopts, false); + + RENDERDOC_FreeEnvironmentModificationList(env); + + return ret; } }; diff --git a/renderdocui/Interop/StaticExports.cs b/renderdocui/Interop/StaticExports.cs index 7f0a262b3..5febfff6f 100644 --- a/renderdocui/Interop/StaticExports.cs +++ b/renderdocui/Interop/StaticExports.cs @@ -62,7 +62,7 @@ namespace renderdoc IntPtr logfile, CaptureOptions opts, bool waitForExit); [DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)] - private static extern UInt32 RENDERDOC_InjectIntoProcess(UInt32 pid, IntPtr logfile, CaptureOptions opts, bool waitForExit); + private static extern UInt32 RENDERDOC_InjectIntoProcess(UInt32 pid, IntPtr env, IntPtr logfile, CaptureOptions opts, bool waitForExit); [DllImport("renderdoc.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr RENDERDOC_CreateTargetControl(IntPtr host, UInt32 ident, IntPtr clientName, bool forceConnection); @@ -191,11 +191,26 @@ namespace renderdoc return ret; } - public static UInt32 InjectIntoProcess(UInt32 pid, string logfile, CaptureOptions opts) + public static UInt32 InjectIntoProcess(UInt32 pid, EnvironmentModification[] env, string logfile, CaptureOptions opts) { IntPtr logfile_mem = CustomMarshal.MakeUTF8String(logfile); - UInt32 ret = RENDERDOC_InjectIntoProcess(pid, logfile_mem, opts, false); + IntPtr env_mem = RENDERDOC_MakeEnvironmentModificationList(env.Length); + + for (int i = 0; i < env.Length; i++) + { + IntPtr var_mem = CustomMarshal.MakeUTF8String(env[i].variable); + IntPtr val_mem = CustomMarshal.MakeUTF8String(env[i].value); + + RENDERDOC_SetEnvironmentModification(env_mem, i, var_mem, val_mem, env[i].type, env[i].separator); + + CustomMarshal.Free(var_mem); + CustomMarshal.Free(val_mem); + } + + UInt32 ret = RENDERDOC_InjectIntoProcess(pid, env_mem, logfile_mem, opts, false); + + RENDERDOC_FreeEnvironmentModificationList(env_mem); CustomMarshal.Free(logfile_mem); diff --git a/renderdocui/Windows/Dialogs/CaptureDialog.cs b/renderdocui/Windows/Dialogs/CaptureDialog.cs index 2623750fa..95530f212 100644 --- a/renderdocui/Windows/Dialogs/CaptureDialog.cs +++ b/renderdocui/Windows/Dialogs/CaptureDialog.cs @@ -260,7 +260,7 @@ namespace renderdocui.Windows.Dialogs #region Callbacks public delegate LiveCapture OnCaptureMethod(string exe, string workingDir, string cmdLine, EnvironmentModification[] env, CaptureOptions opts); - public delegate LiveCapture OnInjectMethod(UInt32 PID, string name, CaptureOptions opts); + public delegate LiveCapture OnInjectMethod(UInt32 PID, EnvironmentModification[] env, string name, CaptureOptions opts); private OnCaptureMethod m_CaptureCallback = null; private OnInjectMethod m_InjectCallback = null; @@ -445,7 +445,7 @@ namespace renderdocui.Windows.Dialogs string name = item.SubItems[1].Text; UInt32 PID = (UInt32)item.Tag; - var live = m_InjectCallback(PID, name, GetSettings().Options); + var live = m_InjectCallback(PID, GetSettings().Environment, name, GetSettings().Options); if (queueFrameCap.Checked && live != null) live.QueueCapture((int)queuedCapFrame.Value); diff --git a/renderdocui/Windows/MainWindow.cs b/renderdocui/Windows/MainWindow.cs index 14a79eede..facba193b 100644 --- a/renderdocui/Windows/MainWindow.cs +++ b/renderdocui/Windows/MainWindow.cs @@ -1022,14 +1022,14 @@ namespace renderdocui.Windows return live; } - private LiveCapture OnInjectTrigger(UInt32 PID, string name, CaptureOptions opts) + private LiveCapture OnInjectTrigger(UInt32 PID, EnvironmentModification[] env, string name, CaptureOptions opts) { if (!PromptCloseLog()) return null; string logfile = m_Core.TempLogFilename(name); - UInt32 ret = StaticExports.InjectIntoProcess(PID, logfile, opts); + UInt32 ret = StaticExports.InjectIntoProcess(PID, env, logfile, opts); if (ret == 0) {