From 2a4d2451ff41b4b7d5660d98c685162f04c4e2ac Mon Sep 17 00:00:00 2001 From: baldurk Date: Thu, 8 Dec 2016 18:23:08 +0000 Subject: [PATCH] Allow 'promoting' back to capture 64-bit from 32-bit, in 64-bit install * A 64-bit install has an x86 subfolder to capture 32-bit programs. If a 32-bit program then launches a 64-bit program we need to 'promote' back to run the original bitness to capture it. * It's still not supported to capture 64-bit in general from just a 32-bit install as the support files aren't included. --- renderdoc/os/win32/win32_process.cpp | 77 +++++++++++++++++++++++++--- renderdoccmd/renderdoccmd.cpp | 8 +-- 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/renderdoc/os/win32/win32_process.cpp b/renderdoc/os/win32/win32_process.cpp index 37a4ab166..d0119e4fe 100644 --- a/renderdoc/os/win32/win32_process.cpp +++ b/renderdoc/os/win32/win32_process.cpp @@ -529,11 +529,14 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, EnvironmentModification *env, return 0; } + bool capalt = false; + #if DISABLED(RDOC_X64) BOOL selfWow64 = FALSE; HANDLE hSelfProcess = GetCurrentProcess(); + // check to see if we're a WoW64 process success = IsWow64Process(hSelfProcess, &selfWow64); CloseHandle(hSelfProcess); @@ -545,22 +548,79 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, EnvironmentModification *env, return 0; } + // we know we're 32-bit, so if the target process is not wow64 + // and we are, it's 64-bit. If we're both not wow64 then we're + // running on 32-bit windows, and if we're both wow64 then we're + // both 32-bit on 64-bit windows. + // + // We don't support capturing 64-bit programs from a 32-bit install + // because it's pointless - a 64-bit install will work for all in + // that case. But we do want to handle the case of: + // 64-bit renderdoc -> 32-bit program (via 32-bit renderdoccmd) + // -> 64-bit program (going back to 64-bit renderdoccmd). + // so we try to see if we're an x86 invoked renderdoccmd in an + // otherwise 64-bit install, and 'promote' back to 64-bit. if(selfWow64 && !isWow64) { - RDCERR("Can't capture x64 process with x86 renderdoc"); - CloseHandle(hProcess); - return 0; + wchar_t *slash = wcsrchr(renderdocPath, L'\\'); + + if(slash && slash > renderdocPath + 4) + { + slash -= 4; + + if(slash && !wcsncmp(slash, L"\\x86", 4)) + { + RDCDEBUG("Promoting back to 64-bit"); + capalt = true; + } + } + + // if we couldn't promote, then bail out. + if(!capalt) + { + RDCERR("Can't capture x64 process with x86 renderdoc"); + + CloseHandle(hProcess); + return 0; + } } #else - // farm off to x86 version - if(isWow64) + // farm off to alternate bitness renderdoccmd.exe + + // if the target process is 'wow64' that means it's 32-bit. + capalt = (isWow64 == TRUE); +#endif + + if(capalt) { +#if ENABLED(RDOC_X64) + // look in a subfolder for x86. + + // remove the filename from the path wchar_t *slash = wcsrchr(renderdocPath, L'\\'); if(slash) *slash = 0; + // append path wcscat_s(renderdocPath, L"\\x86\\renderdoccmd.exe"); +#else + // look upwards on 32-bit to find the parent renderdoccmd. + wchar_t *slash = wcsrchr(renderdocPath, L'\\'); + + // remove the filename + if(slash) + *slash = 0; + + // remove the \\x86 + slash = wcsrchr(renderdocPath, L'\\'); + + if(slash) + *slash = 0; + + // append path + wcscat_s(renderdocPath, L"\\renderdoccmd.exe"); +#endif PROCESS_INFORMATION pi; STARTUPINFO si; @@ -593,9 +653,11 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, EnvironmentModification *env, wstring wdebugLogfile = StringFormat::UTF82Wide(debugLogfile); _snwprintf_s(paramsAlloc, 2047, 2047, - L"\"%ls\" cap32for64 --pid=%d --log=\"%ls\" --debuglog=\"%ls\" --capopts=\"%hs\"", + L"\"%ls\" capaltbit --pid=%d --log=\"%ls\" --debuglog=\"%ls\" --capopts=\"%hs\"", renderdocPath, pid, wlogfile.c_str(), wdebugLogfile.c_str(), optstr.c_str()); + RDCDEBUG("params %ls", paramsAlloc); + paramsAlloc[2047] = 0; wchar_t *commandLine = paramsAlloc; @@ -668,7 +730,7 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, EnvironmentModification *env, if(!retValue) { - RDCERR("Can't spawn x86 renderdoccmd - missing files?"); + RDCERR("Can't spawn alternate bitness renderdoccmd - missing files?"); return 0; } @@ -687,7 +749,6 @@ uint32_t Process::InjectIntoProcess(uint32_t pid, EnvironmentModification *env, return (uint32_t)exitCode; } -#endif InjectDLL(hProcess, renderdocPath); diff --git a/renderdoccmd/renderdoccmd.cpp b/renderdoccmd/renderdoccmd.cpp index b2a23020a..dca583028 100644 --- a/renderdoccmd/renderdoccmd.cpp +++ b/renderdoccmd/renderdoccmd.cpp @@ -542,7 +542,7 @@ struct ReplayCommand : public Command } }; -struct Cap32For64Command : public Command +struct CapAltBitCommand : public Command { virtual void AddOptions(cmdline::parser &parser) { @@ -564,7 +564,7 @@ struct Cap32For64Command : public Command if(rest.size() % 3 != 0) { - std::cerr << "Invalid generated cap32for64 command rest.size() == " << rest.size() << std::endl; + std::cerr << "Invalid generated capaltbit command rest.size() == " << rest.size() << std::endl; return 0; } @@ -626,7 +626,7 @@ struct Cap32For64Command : public Command } else { - std::cerr << "Invalid generated cap32for64 env '" << rest[i * 3 + 0] << std::endl; + std::cerr << "Invalid generated capaltbit env '" << rest[i * 3 + 0] << std::endl; RENDERDOC_FreeEnvironmentModificationList(env); return 0; } @@ -678,7 +678,7 @@ int renderdoccmd(std::vector &argv) add_command("inject", new InjectCommand()); add_command("remoteserver", new RemoteServerCommand()); add_command("replay", new ReplayCommand()); - add_command("cap32for64", new Cap32For64Command()); + add_command("capaltbit", new CapAltBitCommand()); if(argv.size() <= 1) {