From 6afe7b7cd18a72dc5e1d02254cae3b42e9b11782 Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Sun, 16 Nov 2014 16:38:48 +0000
Subject: [PATCH] Speculative change to prevent infinite loop of
 renderdoccmd.exe

* There was a report of renderdoccmd.exe doing --cap32for64 in an infinite
  loop on itself (each process trying to inject the previous). I'm not sure
  how this could get started, but assuming this was happening automatically
  for 'hook into children', try to break the infinite loop by refusing to
  inject into our own exes.
---
 renderdoc/hooks/sys_win32_hooks.cpp | 79 ++++++++++++++++++++++++++---
 1 file changed, 71 insertions(+), 8 deletions(-)

diff --git a/renderdoc/hooks/sys_win32_hooks.cpp b/renderdoc/hooks/sys_win32_hooks.cpp
index 5e18663a1..dec7b422b 100644
--- a/renderdoc/hooks/sys_win32_hooks.cpp
+++ b/renderdoc/hooks/sys_win32_hooks.cpp
@@ -25,6 +25,7 @@
 
 #include "core/core.h"
 #include "api/replay/renderdoc_replay.h"
+#include "common/string_utils.h"
 
 #include "hooks.h"
 
@@ -129,11 +130,42 @@ class SysHook : LibraryHook
 			{
 				RDCDEBUG("Intercepting CreateProcessA");
 
-				// inherit logfile and capture options
-				uint32_t ident = RENDERDOC_InjectIntoProcess(lpProcessInformation->dwProcessId,
-										RenderDoc::Inst().GetLogFile(), &RenderDoc::Inst().GetCaptureOptions(), false);
+				bool inject = true;
 
-				RenderDoc::Inst().AddChildProcess((uint32_t)lpProcessInformation->dwProcessId, ident);
+				// sanity check to make sure we're not going to go into an infinity loop injecting into ourselves.
+				if(lpApplicationName)
+				{
+					string app = lpApplicationName;
+					app = strlower(app);
+
+					if(app.find("renderdoccmd.exe") != string::npos ||
+					   app.find("renderdocui.vshost.exe") != string::npos ||
+					   app.find("renderdocui.exe") != string::npos)
+					{
+						inject = false;
+					}
+				}
+				if(lpCommandLine)
+				{
+					string cmd = lpCommandLine;
+					cmd = strlower(cmd);
+
+					if(cmd.find("renderdoccmd.exe") != string::npos ||
+					   cmd.find("renderdocui.vshost.exe") != string::npos ||
+					   cmd.find("renderdocui.exe") != string::npos)
+					{
+						inject = false;
+					}
+				}
+
+				if(inject)
+				{
+					// inherit logfile and capture options
+					uint32_t ident = RENDERDOC_InjectIntoProcess(lpProcessInformation->dwProcessId,
+						RenderDoc::Inst().GetLogFile(), &RenderDoc::Inst().GetCaptureOptions(), false);
+
+					RenderDoc::Inst().AddChildProcess((uint32_t)lpProcessInformation->dwProcessId, ident);
+				}
 			}
 
 			ResumeThread(lpProcessInformation->hThread);
@@ -185,11 +217,42 @@ class SysHook : LibraryHook
 			{
 				RDCDEBUG("Intercepting CreateProcessW");
 
-				// inherit logfile and capture options
-				uint32_t ident = RENDERDOC_InjectIntoProcess(lpProcessInformation->dwProcessId,
-										RenderDoc::Inst().GetLogFile(), &RenderDoc::Inst().GetCaptureOptions(), false);
+				bool inject = true;
 
-				RenderDoc::Inst().AddChildProcess((uint32_t)lpProcessInformation->dwProcessId, ident);
+				// sanity check to make sure we're not going to go into an infinity loop injecting into ourselves.
+				if(lpApplicationName)
+				{
+					wstring app = lpApplicationName;
+					app = strlower(app);
+
+					if(app.find(L"renderdoccmd.exe") != wstring::npos ||
+					   app.find(L"renderdocui.vshost.exe") != wstring::npos ||
+					   app.find(L"renderdocui.exe") != wstring::npos)
+					{
+						inject = false;
+					}
+				}
+				if(lpCommandLine)
+				{
+					wstring cmd = lpCommandLine;
+					cmd = strlower(cmd);
+
+					if(cmd.find(L"renderdoccmd.exe") != wstring::npos ||
+					   cmd.find(L"renderdocui.vshost.exe") != wstring::npos ||
+					   cmd.find(L"renderdocui.exe") != wstring::npos)
+					{
+						inject = false;
+					}
+				}
+
+				if(inject)
+				{
+					// inherit logfile and capture options
+					uint32_t ident = RENDERDOC_InjectIntoProcess(lpProcessInformation->dwProcessId,
+											RenderDoc::Inst().GetLogFile(), &RenderDoc::Inst().GetCaptureOptions(), false);
+
+					RenderDoc::Inst().AddChildProcess((uint32_t)lpProcessInformation->dwProcessId, ident);
+				}
 			}
 			
 			ResumeThread(lpProcessInformation->hThread);