From 2bc81b861774f2371d62443c3280df765ff7c3f1 Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Mon, 13 Aug 2018 18:27:15 +0100
Subject: [PATCH] Don't try to recreate breakpad server if it's still running

---
 renderdoc/core/core.cpp        |   2 -
 renderdoc/core/core.h          |   3 -
 renderdoc/core/crash_handler.h | 117 +++++++++++++++++++--------------
 3 files changed, 67 insertions(+), 55 deletions(-)

diff --git a/renderdoc/core/core.cpp b/renderdoc/core/core.cpp
index 6488b9470..74a6dac84 100644
--- a/renderdoc/core/core.cpp
+++ b/renderdoc/core/core.cpp
@@ -181,8 +181,6 @@ RenderDoc &RenderDoc::Inst()
 
 void RenderDoc::RecreateCrashHandler()
 {
-  UnloadCrashHandler();
-
 #if ENABLED(RDOC_CRASH_HANDLER)
   m_ExHandler = new CrashHandler(m_ExHandler);
 #endif
diff --git a/renderdoc/core/core.h b/renderdoc/core/core.h
index c0c3b5b1f..fb85a3700 100644
--- a/renderdoc/core/core.h
+++ b/renderdoc/core/core.h
@@ -52,9 +52,6 @@ bool is_exr_file(FILE *f);
 struct ICrashHandler
 {
   virtual ~ICrashHandler() {}
-  virtual void WriteMinidump() = 0;
-  virtual void WriteMinidump(void *data) = 0;
-
   virtual void RegisterMemoryRegion(void *mem, size_t size) = 0;
   virtual void UnregisterMemoryRegion(void *mem) = 0;
 };
diff --git a/renderdoc/core/crash_handler.h b/renderdoc/core/crash_handler.h
index 8c5165bac..9f0acdf2a 100644
--- a/renderdoc/core/crash_handler.h
+++ b/renderdoc/core/crash_handler.h
@@ -42,7 +42,15 @@ public:
     google_breakpad::AppMemoryList mem;
 
     if(existing)
+    {
       mem = ((CrashHandler *)existing)->m_ExHandler->QueryRegisteredAppMemory();
+    }
+    else
+    {
+      CreateCrashHandlingServer();
+    }
+
+    _CrtSetReportMode(_CRT_ASSERT, 0);
 
     SAFE_DELETE(existing);
 
@@ -58,51 +66,6 @@ public:
 
     MINIDUMP_TYPE dumpType = MINIDUMP_TYPE(MiniDumpNormal | MiniDumpWithIndirectlyReferencedMemory);
 
-    {
-      PROCESS_INFORMATION pi;
-      STARTUPINFOW si;
-      RDCEraseEl(pi);
-      RDCEraseEl(si);
-
-      HANDLE waitEvent = CreateEventA(NULL, TRUE, FALSE, "RENDERDOC_CRASHHANDLE");
-
-      wchar_t radpath[MAX_PATH] = {0};
-      GetModuleFileNameW(GetModuleHandleA("renderdoc.dll"), radpath, MAX_PATH - 1);
-
-      wchar_t *slash = wcsrchr(radpath, L'\\');
-
-      if(slash)
-      {
-        *slash = 0;
-      }
-      else
-      {
-        slash = wcsrchr(radpath, L'/');
-
-        if(slash)
-          *slash = 0;
-        else
-        {
-          radpath[0] = L'.';
-          radpath[1] = 0;
-        }
-      }
-
-      wstring cmdline = L"\"";
-      cmdline += radpath;
-      cmdline += L"/renderdoccmd.exe\" crashhandle";
-
-      wchar_t *paramsAlloc = new wchar_t[512];
-
-      wcscpy_s(paramsAlloc, 511, cmdline.c_str());
-
-      CreateProcessW(NULL, paramsAlloc, NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi);
-
-      WaitForSingleObject(waitEvent, 2000);
-
-      CloseHandle(waitEvent);
-    }
-
     static google_breakpad::CustomInfoEntry breakpadCustomInfo[] = {
         google_breakpad::CustomInfoEntry(L"version", L""),
         google_breakpad::CustomInfoEntry(L"logpath", L""),
@@ -121,24 +84,78 @@ public:
     google_breakpad::CustomClientInfo custom = {&breakpadCustomInfo[0],
                                                 ARRAY_COUNT(breakpadCustomInfo)};
 
-    _CrtSetReportMode(_CRT_ASSERT, 0);
     m_ExHandler = new google_breakpad::ExceptionHandler(
         dumpFolder.c_str(), NULL, NULL, NULL, google_breakpad::ExceptionHandler::HANDLER_ALL,
         dumpType, L"\\\\.\\pipe\\RenderDocBreakpadServer", &custom);
 
+    if(!m_ExHandler->IsOutOfProcess())
+    {
+      RDCWARN("Couldn't find existing breakpad server");
+
+      SAFE_DELETE(m_ExHandler);
+
+      CreateCrashHandlingServer();
+
+      m_ExHandler = new google_breakpad::ExceptionHandler(
+          dumpFolder.c_str(), NULL, NULL, NULL, google_breakpad::ExceptionHandler::HANDLER_ALL,
+          dumpType, L"\\\\.\\pipe\\RenderDocBreakpadServer", &custom);
+    }
+
     m_ExHandler->set_handle_debug_exceptions(true);
 
     for(size_t i = 0; i < mem.size(); i++)
       m_ExHandler->RegisterAppMemory((void *)mem[i].ptr, mem[i].length);
   }
 
-  virtual ~CrashHandler() { SAFE_DELETE(m_ExHandler); }
-  void WriteMinidump() { m_ExHandler->WriteMinidump(); }
-  void WriteMinidump(void *data)
+  void CreateCrashHandlingServer()
   {
-    m_ExHandler->WriteMinidumpForException((EXCEPTION_POINTERS *)data);
+    RDCLOG("Creating crash-handling server");
+
+    PROCESS_INFORMATION pi;
+    STARTUPINFOW si;
+    RDCEraseEl(pi);
+    RDCEraseEl(si);
+
+    HANDLE waitEvent = CreateEventA(NULL, TRUE, FALSE, "RENDERDOC_CRASHHANDLE");
+
+    wchar_t radpath[MAX_PATH] = {0};
+    GetModuleFileNameW(GetModuleHandleA("renderdoc.dll"), radpath, MAX_PATH - 1);
+
+    wchar_t *slash = wcsrchr(radpath, L'\\');
+
+    if(slash)
+    {
+      *slash = 0;
+    }
+    else
+    {
+      slash = wcsrchr(radpath, L'/');
+
+      if(slash)
+        *slash = 0;
+      else
+      {
+        radpath[0] = L'.';
+        radpath[1] = 0;
+      }
+    }
+
+    wstring cmdline = L"\"";
+    cmdline += radpath;
+    cmdline += L"/renderdoccmd.exe\" crashhandle";
+
+    wchar_t *paramsAlloc = new wchar_t[512];
+
+    wcscpy_s(paramsAlloc, 511, cmdline.c_str());
+
+    CreateProcessW(NULL, paramsAlloc, NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi);
+
+    WaitForSingleObject(waitEvent, 2000);
+
+    CloseHandle(waitEvent);
   }
 
+  virtual ~CrashHandler() { SAFE_DELETE(m_ExHandler); }
   void RegisterMemoryRegion(void *mem, size_t size) { m_ExHandler->RegisterAppMemory(mem, size); }
   void UnregisterMemoryRegion(void *mem) { m_ExHandler->UnregisterAppMemory(mem); }
 private: