lee/renderdoc
1
0
Fork 0
mirror of https://github.com/baldurk/renderdoc.git synced 2026-05-29 21:30:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Speculative change to prevent infinite loop of renderdoccmd.exe

Browse Source 
* There was a report of renderdoccmd.exe doing --cap32for64 in an infinite
  loop on itself (each process trying to inject the previous). I'm not sure
  how this could get started, but assuming this was happening automatically
  for 'hook into children', try to break the infinite loop by refusing to
  inject into our own exes.
This commit is contained in:
baldurk
2014-11-16 16:38:48 +00:00
parent 643447f521
commit 6afe7b7cd1
1 changed files with 71 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
renderdoc/hooks/sys_win32_hooks.cpp
+71 -8
View File
@@ -25,6 +25,7 @@
#include "core/core.h"
#include "api/replay/renderdoc_replay.h"
#include "common/string_utils.h"
#include "hooks.h"
@@ -129,11 +130,42 @@ class SysHook : LibraryHook
{
RDCDEBUG("Intercepting CreateProcessA");
// inherit logfile and capture options
uint32_t ident = RENDERDOC_InjectIntoProcess(lpProcessInformation->dwProcessId,
RenderDoc::Inst().GetLogFile(), &RenderDoc::Inst().GetCaptureOptions(), false);
bool inject = true;
RenderDoc::Inst().AddChildProcess((uint32_t)lpProcessInformation->dwProcessId, ident);
// sanity check to make sure we're not going to go into an infinity loop injecting into ourselves.
if(lpApplicationName)
{
string app = lpApplicationName;
app = strlower(app);
if(app.find("renderdoccmd.exe") != string::npos ||
app.find("renderdocui.vshost.exe") != string::npos ||
app.find("renderdocui.exe") != string::npos)
{
inject = false;
}
}
if(lpCommandLine)
{
string cmd = lpCommandLine;
cmd = strlower(cmd);
if(cmd.find("renderdoccmd.exe") != string::npos ||
cmd.find("renderdocui.vshost.exe") != string::npos ||
cmd.find("renderdocui.exe") != string::npos)
{
inject = false;
}
}
if(inject)
{
// inherit logfile and capture options
uint32_t ident = RENDERDOC_InjectIntoProcess(lpProcessInformation->dwProcessId,
RenderDoc::Inst().GetLogFile(), &RenderDoc::Inst().GetCaptureOptions(), false);
RenderDoc::Inst().AddChildProcess((uint32_t)lpProcessInformation->dwProcessId, ident);
}
}
ResumeThread(lpProcessInformation->hThread);
@@ -185,11 +217,42 @@ class SysHook : LibraryHook
{
RDCDEBUG("Intercepting CreateProcessW");
// inherit logfile and capture options
uint32_t ident = RENDERDOC_InjectIntoProcess(lpProcessInformation->dwProcessId,
RenderDoc::Inst().GetLogFile(), &RenderDoc::Inst().GetCaptureOptions(), false);
bool inject = true;
RenderDoc::Inst().AddChildProcess((uint32_t)lpProcessInformation->dwProcessId, ident);
// sanity check to make sure we're not going to go into an infinity loop injecting into ourselves.
if(lpApplicationName)
{
wstring app = lpApplicationName;
app = strlower(app);
if(app.find(L"renderdoccmd.exe") != wstring::npos ||
app.find(L"renderdocui.vshost.exe") != wstring::npos ||
app.find(L"renderdocui.exe") != wstring::npos)
{
inject = false;
}
}
if(lpCommandLine)
{
wstring cmd = lpCommandLine;
cmd = strlower(cmd);
if(cmd.find(L"renderdoccmd.exe") != wstring::npos ||
cmd.find(L"renderdocui.vshost.exe") != wstring::npos ||
cmd.find(L"renderdocui.exe") != wstring::npos)
{
inject = false;
}
}
if(inject)
{
// inherit logfile and capture options
uint32_t ident = RENDERDOC_InjectIntoProcess(lpProcessInformation->dwProcessId,
RenderDoc::Inst().GetLogFile(), &RenderDoc::Inst().GetCaptureOptions(), false);
RenderDoc::Inst().AddChildProcess((uint32_t)lpProcessInformation->dwProcessId, ident);
}
}
ResumeThread(lpProcessInformation->hThread);