Add process for programs to voluntarily remove renderdoc. Refs #115

* To remove renderdoc (e.g. if the API version doesn't match what is
  expected), user code can call RENDERDOC_Shutdown() to remove the running
  thread and undo any library hooks, then unload the dynamic library.
* This still isn't recommended as there could be unexpected side-effects,
  and it will definitely break if this happens after any APIs have been
  initialised, but it's an option.

renderdoc/api/app/renderdoc_app.h

@@ -235,6 +235,9 @@ enum InAppOverlay
 extern "C" RENDERDOC_API int RENDERDOC_CC RENDERDOC_GetAPIVersion();
 typedef int (RENDERDOC_CC *pRENDERDOC_GetAPIVersion)();
 
+extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_Shutdown();
+typedef void (RENDERDOC_CC *pRENDERDOC_Shutdown)();
+
 extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_SetLogFile(const char *logfile);
 typedef void (RENDERDOC_CC *pRENDERDOC_SetLogFile)(const char *logfile);
 

renderdoc/core/core.cpp

@@ -265,16 +265,37 @@ RenderDoc::~RenderDoc()
 	
 	FileIO::Delete(m_LoggingFilename.c_str());
 
-	m_RemoteServerThreadShutdown = true;
-	// don't join, just close the thread, as we can't wait while in the middle of module unloading
-	Threading::CloseThread(m_RemoteThread);
-	m_RemoteThread = 0;
+	if(m_RemoteThread)
+	{
+		m_RemoteServerThreadShutdown = true;
+		// don't join, just close the thread, as we can't wait while in the middle of module unloading
+		Threading::CloseThread(m_RemoteThread);
+		m_RemoteThread = 0;
+	}
 
 	Network::Shutdown();
 
 	FileIO::Delete(m_LoggingFilename.c_str());
 }
 
+void RenderDoc::Shutdown()
+{
+	if(m_ExHandler)
+	{
+		UnloadCrashHandler();
+	}
+	
+	if(m_RemoteThread)
+	{
+		// explicitly wait for thread to shutdown, this call is not from module unloading and
+		// we want to be sure everything is gone before we remove our module & hooks
+		m_RemoteServerThreadShutdown = true;
+		Threading::JoinThread(m_RemoteThread);
+		Threading::CloseThread(m_RemoteThread);
+		m_RemoteThread = 0;
+	}
+}
+
 void RenderDoc::StartFrameCapture(void *wnd)
 {
 	auto it = m_WindowFrameCapturers.find(wnd);

renderdoc/core/core.h

@@ -169,6 +169,7 @@ class RenderDoc
 		const char *GetCurrentTarget() const { return m_Target.c_str(); }
 
 		void Initialise();
+		void Shutdown();
 
 		void SetReplayApp(bool replay) { m_Replay = replay; }
 		bool IsReplayApp() const { return m_Replay; }

renderdoc/hooks/hooks.cpp

@@ -56,6 +56,13 @@ void LibraryHooks::CreateHooks()
 	HOOKS_END();
 }
 
+void LibraryHooks::RemoveHooks()
+{
+	if(m_HooksRemoved) return;
+	m_HooksRemoved = true;
+	HOOKS_REMOVE();
+}
+
 void LibraryHooks::EnableHooks(bool enable)
 {
 	RDCDEBUG("%s hooks!", enable ? "Enabling" : "Disabling");

renderdoc/hooks/hooks.h

@@ -72,6 +72,7 @@ class Hook
 
 #define HOOKS_BEGIN() Win32_IAT_BeginHooks()
 #define HOOKS_END() Win32_IAT_EndHooks()
+#define HOOKS_REMOVE() Win32_IAT_RemoveHooks()
 
 #elif defined(LINUX)
 
@@ -80,6 +81,7 @@ class Hook
 
 #define HOOKS_BEGIN()
 #define HOOKS_END()
+#define HOOKS_REMOVE()
 
 #else
 
@@ -101,13 +103,17 @@ struct LibraryHook
 class LibraryHooks
 {
 	public:
+		LibraryHooks() : m_HooksRemoved(false) {}
 		static LibraryHooks &GetInstance();
 		void RegisterHook(const char *libName, LibraryHook *hook);
 		void CreateHooks();
 		void EnableHooks(bool enable);
+		void RemoveHooks();
 
 	private:
 		typedef map<const char *, LibraryHook *> HookMap;
 
+		bool m_HooksRemoved;
+
 		HookMap m_Hooks;
 };

renderdoc/os/win32/win32_hook.cpp

@@ -36,6 +36,10 @@
 using std::vector;
 using std::map;
 
+// map from address of IAT entry, to original contents
+map<void **, void *> s_InstalledHooks;
+Threading::CriticalSection installedLock;
+
 struct FunctionHook
 {
 	FunctionHook(const char *f, void **o, void *d)
@@ -54,6 +58,12 @@ struct FunctionHook
 		if(*IATentry == hookptr)
 			return false;
 
+		{
+			SCOPED_LOCK(installedLock);
+			if(s_InstalledHooks.find(IATentry) == s_InstalledHooks.end())
+				s_InstalledHooks[IATentry] = *IATentry;
+		}
+
 		BOOL success = TRUE;
 
 		success = VirtualProtect(IATentry, sizeof(void*), PAGE_READWRITE, &oldProtection);
@@ -390,6 +400,33 @@ void Win32_IAT_EndHooks()
 	HookAllModules();
 }
 
+void Win32_IAT_RemoveHooks()
+{
+	for(auto it=s_InstalledHooks.begin(); it != s_InstalledHooks.end(); ++it)
+	{
+		DWORD oldProtection = PAGE_EXECUTE;
+		BOOL success = TRUE;
+		
+		void **IATentry = it->first;
+
+		success = VirtualProtect(IATentry, sizeof(void*), PAGE_READWRITE, &oldProtection);
+		if(!success)
+		{
+			RDCERR("Failed to make IAT entry writeable 0x%p", IATentry);
+			continue;
+		}
+		
+		*IATentry = it->second;
+
+		success = VirtualProtect(IATentry, sizeof(void*), oldProtection, &oldProtection);
+		if(!success)
+		{
+			RDCERR("Failed to restore IAT entry protection 0x%p", IATentry);
+			continue;
+		}
+	}
+}
+
 bool Win32_IAT_Hook(void **orig_function_ptr, const char *module_name, const char *function, void *destination_function_ptr)
 {
 	if(!_stricmp(module_name, "kernel32.dll"))

renderdoc/os/win32/win32_hook.h

@@ -27,3 +27,4 @@
 bool Win32_IAT_Hook(void **orig_function_ptr, const char *module_name, const char *function, void *destination_function_ptr);
 void Win32_IAT_BeginHooks();
 void Win32_IAT_EndHooks();
+void Win32_IAT_RemoveHooks();

renderdoc/replay/entry_points.cpp

@@ -27,6 +27,7 @@
 #include "maths/formatpacking.h"
 #include "serialise/serialiser.h"
 #include "core/core.h"
+#include "hooks/hooks.h"
 #include "replay/replay_renderer.h"
 #include "api/replay/renderdoc_replay.h"
 
@@ -90,6 +91,13 @@ int RENDERDOC_CC RENDERDOC_GetAPIVersion()
 	return RENDERDOC_API_VERSION;
 }
 
+extern "C" RENDERDOC_API
+void RENDERDOC_CC RENDERDOC_Shutdown()
+{
+	RenderDoc::Inst().Shutdown();
+	LibraryHooks::GetInstance().RemoveHooks();
+}
+
 extern "C" RENDERDOC_API
 void RENDERDOC_CC RENDERDOC_LogText(const char *text)
 {