From e1fe211f3c0a51e035b6de49b70dfe634f5a41c2 Mon Sep 17 00:00:00 2001 From: baldurk Date: Thu, 30 Jan 2020 13:02:23 +0000 Subject: [PATCH] Rename Shutdown to RemoveHooks in in-application API. Version is 1.4.1 * This is a binary and source backwards compatible change. --- docs/in_application_api.rst | 12 ++++++-- renderdoc/api/app/renderdoc_app.h | 45 ++++++++++++++++++---------- renderdoc/core/core.cpp | 2 +- renderdoc/core/core.h | 2 +- renderdoc/driver/vulkan/vk_layer.cpp | 8 +++++ renderdoc/hooks/hooks.cpp | 6 ++++ renderdoc/hooks/hooks.h | 2 ++ renderdoc/os/win32/win32_hook.cpp | 2 ++ renderdoc/replay/app_api.cpp | 37 ++++++++++++----------- 9 files changed, 78 insertions(+), 38 deletions(-) diff --git a/docs/in_application_api.rst b/docs/in_application_api.rst index de0c586c1..41ad3b93e 100644 --- a/docs/in_application_api.rst +++ b/docs/in_application_api.rst @@ -287,9 +287,17 @@ This is not supported on Linux or Android due to the differing methods of hookin :param uint32_t And: is a 32-bit value the mask is binary-AND'd with before processing ``Or``. :param uint32_t Or: is a 32-bit value the mask is binary-OR'd with after processing ``And``. -.. cpp:function:: void Shutdown() +.. cpp:function:: void RemoveHooks() - This function will attempt to shut down and remove RenderDoc and its hooks from the target process. It must be called as early as possible in the process, and will have undefined results if any graphics API functions have been called. + This function will attempt to remove RenderDoc and its hooks from the target process. It must be called as early as possible in the process, and will have undefined results if any graphics API functions have been called. + +.. note:: + + This process is only possible on Windows, and even then it is not well defined so may not be possible in all circumstances. This function is provided at your own risk. + +.. note:: + + This function was renamed, in earlier versions of the API it was declared as ``Shutdown``. This rename is backwards compatible as the function signature did not change. .. cpp:function:: void UnloadCrashHandler() diff --git a/renderdoc/api/app/renderdoc_app.h b/renderdoc/api/app/renderdoc_app.h index 725d30b65..075306c04 100644 --- a/renderdoc/api/app/renderdoc_app.h +++ b/renderdoc/api/app/renderdoc_app.h @@ -349,13 +349,17 @@ typedef uint32_t(RENDERDOC_CC *pRENDERDOC_GetOverlayBits)(); // sets the overlay bits with an and & or mask typedef void(RENDERDOC_CC *pRENDERDOC_MaskOverlayBits)(uint32_t And, uint32_t Or); -// this function will attempt to shut down RenderDoc. +// this function will attempt to remove RenderDoc's hooks in the application. // -// Note: that this will only work correctly if done immediately after -// the dll is loaded, before any API work happens. RenderDoc will remove its +// Note: that this can only work correctly if done immediately after +// the module is loaded, before any API work happens. RenderDoc will remove its // injected hooks and shut down. Behaviour is undefined if this is called -// after any API functions have been called. -typedef void(RENDERDOC_CC *pRENDERDOC_Shutdown)(); +// after any API functions have been called, and there is still no guarantee of +// success. +typedef void(RENDERDOC_CC *pRENDERDOC_RemoveHooks)(); + +// DEPRECATED: compatibility for code compiled against pre-1.4.1 headers. +typedef pRENDERDOC_RemoveHooks pRENDERDOC_Shutdown; // This function will unload RenderDoc's crash handler. // @@ -543,6 +547,7 @@ typedef enum RENDERDOC_Version { eRENDERDOC_API_Version_1_2_0 = 10200, // RENDERDOC_API_1_2_0 = 1 02 00 eRENDERDOC_API_Version_1_3_0 = 10300, // RENDERDOC_API_1_3_0 = 1 03 00 eRENDERDOC_API_Version_1_4_0 = 10400, // RENDERDOC_API_1_4_0 = 1 04 00 + eRENDERDOC_API_Version_1_4_1 = 10401, // RENDERDOC_API_1_4_1 = 1 04 01 } RENDERDOC_Version; // API version changelog: @@ -568,8 +573,9 @@ typedef enum RENDERDOC_Version { // 0xdddddddd of uninitialised buffer contents. // 1.4.0 - Added feature: DiscardFrameCapture() to discard a frame capture in progress and stop // capturing without saving anything to disk. +// 1.4.1 - Refactor: Renamed Shutdown to RemoveHooks to better clarify what is happening -typedef struct RENDERDOC_API_1_4_0 +typedef struct RENDERDOC_API_1_4_1 { pRENDERDOC_GetAPIVersion GetAPIVersion; @@ -585,7 +591,13 @@ typedef struct RENDERDOC_API_1_4_0 pRENDERDOC_GetOverlayBits GetOverlayBits; pRENDERDOC_MaskOverlayBits MaskOverlayBits; - pRENDERDOC_Shutdown Shutdown; + // Shutdown was renamed to RemoveHooks in 1.4.1. + // These unions allow old code to continue compiling without changes + union + { + pRENDERDOC_Shutdown Shutdown; + pRENDERDOC_RemoveHooks RemoveHooks; + }; pRENDERDOC_UnloadCrashHandler UnloadCrashHandler; // Get/SetLogFilePathTemplate was renamed to Get/SetCaptureFilePathTemplate in 1.1.2. @@ -635,16 +647,17 @@ typedef struct RENDERDOC_API_1_4_0 // new function in 1.4.0 pRENDERDOC_DiscardFrameCapture DiscardFrameCapture; -} RENDERDOC_API_1_4_0; +} RENDERDOC_API_1_4_1; -typedef RENDERDOC_API_1_4_0 RENDERDOC_API_1_0_0; -typedef RENDERDOC_API_1_4_0 RENDERDOC_API_1_0_1; -typedef RENDERDOC_API_1_4_0 RENDERDOC_API_1_0_2; -typedef RENDERDOC_API_1_4_0 RENDERDOC_API_1_1_0; -typedef RENDERDOC_API_1_4_0 RENDERDOC_API_1_1_1; -typedef RENDERDOC_API_1_4_0 RENDERDOC_API_1_1_2; -typedef RENDERDOC_API_1_4_0 RENDERDOC_API_1_2_0; -typedef RENDERDOC_API_1_4_0 RENDERDOC_API_1_3_0; +typedef RENDERDOC_API_1_4_1 RENDERDOC_API_1_0_0; +typedef RENDERDOC_API_1_4_1 RENDERDOC_API_1_0_1; +typedef RENDERDOC_API_1_4_1 RENDERDOC_API_1_0_2; +typedef RENDERDOC_API_1_4_1 RENDERDOC_API_1_1_0; +typedef RENDERDOC_API_1_4_1 RENDERDOC_API_1_1_1; +typedef RENDERDOC_API_1_4_1 RENDERDOC_API_1_1_2; +typedef RENDERDOC_API_1_4_1 RENDERDOC_API_1_2_0; +typedef RENDERDOC_API_1_4_1 RENDERDOC_API_1_3_0; +typedef RENDERDOC_API_1_4_1 RENDERDOC_API_1_4_0; ////////////////////////////////////////////////////////////////////////////////////////////////// // RenderDoc API entry point diff --git a/renderdoc/core/core.cpp b/renderdoc/core/core.cpp index fae8f1bbc..f3b27fe6e 100644 --- a/renderdoc/core/core.cpp +++ b/renderdoc/core/core.cpp @@ -502,7 +502,7 @@ RenderDoc::~RenderDoc() StringFormat::Shutdown(); } -void RenderDoc::Shutdown() +void RenderDoc::RemoveHooks() { if(m_ExHandler) { diff --git a/renderdoc/core/core.h b/renderdoc/core/core.h index c9d908cc3..3dc6da711 100644 --- a/renderdoc/core/core.h +++ b/renderdoc/core/core.h @@ -407,7 +407,7 @@ public: const char *GetCaptureFileTemplate() const { return m_CaptureFileTemplate.c_str(); } const char *GetCurrentTarget() const { return m_Target.c_str(); } void Initialise(); - void Shutdown(); + void RemoveHooks(); uint64_t GetMicrosecondTimestamp() { return uint64_t(m_Timer.GetMicroseconds()); } const GlobalEnvironment &GetGlobalEnvironment() { return m_GlobalEnv; } diff --git a/renderdoc/driver/vulkan/vk_layer.cpp b/renderdoc/driver/vulkan/vk_layer.cpp index 6816c6353..fcd9bac4d 100644 --- a/renderdoc/driver/vulkan/vk_layer.cpp +++ b/renderdoc/driver/vulkan/vk_layer.cpp @@ -75,6 +75,14 @@ class VulkanHook : LibraryHook OptionsUpdated(); } + void RemoveHooks() + { + // unset the vulkan layer environment variable + Process::RegisterEnvironmentModification(EnvironmentModification( + EnvMod::Set, EnvSep::NoSep, "ENABLE_VULKAN_RENDERDOC_CAPTURE", "0")); + Process::ApplyEnvironmentModification(); + } + void OptionsUpdated() { if(RenderDoc::Inst().GetCaptureOptions().apiValidation) diff --git a/renderdoc/hooks/hooks.cpp b/renderdoc/hooks/hooks.cpp index 0e2d9b4b3..725f085b7 100644 --- a/renderdoc/hooks/hooks.cpp +++ b/renderdoc/hooks/hooks.cpp @@ -47,6 +47,12 @@ void LibraryHooks::RegisterHooks() EndHookRegistration(); } +void LibraryHooks::RemoveHookCallbacks() +{ + for(LibraryHook *lib : LibList()) + lib->RemoveHooks(); +} + void LibraryHooks::OptionsUpdated() { for(LibraryHook *lib : LibList()) diff --git a/renderdoc/hooks/hooks.h b/renderdoc/hooks/hooks.h index 472b88bc5..52b3b3806 100644 --- a/renderdoc/hooks/hooks.h +++ b/renderdoc/hooks/hooks.h @@ -147,6 +147,7 @@ public: // generic, implemented in hooks.cpp to iterate over all registered libraries static void RegisterHooks(); static void OptionsUpdated(); + static void RemoveHookCallbacks(); // platform specific implementations @@ -183,6 +184,7 @@ struct LibraryHook LibraryHook(); virtual void RegisterHooks() = 0; virtual void OptionsUpdated() {} + virtual void RemoveHooks() {} private: friend class LibraryHooks; diff --git a/renderdoc/os/win32/win32_hook.cpp b/renderdoc/os/win32/win32_hook.cpp index f23388b37..d470107f8 100644 --- a/renderdoc/os/win32/win32_hook.cpp +++ b/renderdoc/os/win32/win32_hook.cpp @@ -913,6 +913,8 @@ void LibraryHooks::Refresh() void LibraryHooks::RemoveHooks() { + LibraryHooks::RemoveHookCallbacks(); + for(auto it = s_InstalledHooks.begin(); it != s_InstalledHooks.end(); ++it) { DWORD oldProtection = PAGE_EXECUTE; diff --git a/renderdoc/replay/app_api.cpp b/renderdoc/replay/app_api.cpp index 9743fc331..d65e27a4b 100644 --- a/renderdoc/replay/app_api.cpp +++ b/renderdoc/replay/app_api.cpp @@ -51,9 +51,9 @@ static void MaskOverlayBits(uint32_t And, uint32_t Or) RenderDoc::Inst().MaskOverlayBits(And, Or); } -static void Shutdown() +static void RemoveHooks() { - RenderDoc::Inst().Shutdown(); + RenderDoc::Inst().RemoveHooks(); LibraryHooks::RemoveHooks(); } @@ -245,22 +245,22 @@ int RENDERDOC_CC SetCaptureOptionF32(RENDERDOC_CaptureOption opt, float val); uint32_t RENDERDOC_CC GetCaptureOptionU32(RENDERDOC_CaptureOption opt); float RENDERDOC_CC GetCaptureOptionF32(RENDERDOC_CaptureOption opt); -void RENDERDOC_CC GetAPIVersion_1_4_0(int *major, int *minor, int *patch) +void RENDERDOC_CC GetAPIVersion_1_4_1(int *major, int *minor, int *patch) { if(major) *major = 1; if(minor) *minor = 4; if(patch) - *patch = 0; + *patch = 1; } -RENDERDOC_API_1_4_0 api_1_4_0; -void Init_1_4_0() +RENDERDOC_API_1_4_1 api_1_4_1; +void Init_1_4_1() { - RENDERDOC_API_1_4_0 &api = api_1_4_0; + RENDERDOC_API_1_4_1 &api = api_1_4_1; - api.GetAPIVersion = &GetAPIVersion_1_4_0; + api.GetAPIVersion = &GetAPIVersion_1_4_1; api.SetCaptureOptionU32 = &SetCaptureOptionU32; api.SetCaptureOptionF32 = &SetCaptureOptionF32; @@ -274,7 +274,7 @@ void Init_1_4_0() api.GetOverlayBits = &GetOverlayBits; api.MaskOverlayBits = &MaskOverlayBits; - api.Shutdown = &Shutdown; + api.RemoveHooks = &RemoveHooks; api.UnloadCrashHandler = &UnloadCrashHandler; api.SetCaptureFilePathTemplate = &SetCaptureFilePathTemplate; @@ -325,15 +325,16 @@ extern "C" RENDERDOC_API int RENDERDOC_CC RENDERDOC_GetAPI(RENDERDOC_Version ver ret = 1; \ } - API_VERSION_HANDLE(1_0_0, 1_4_0); - API_VERSION_HANDLE(1_0_1, 1_4_0); - API_VERSION_HANDLE(1_0_2, 1_4_0); - API_VERSION_HANDLE(1_1_0, 1_4_0); - API_VERSION_HANDLE(1_1_1, 1_4_0); - API_VERSION_HANDLE(1_1_2, 1_4_0); - API_VERSION_HANDLE(1_2_0, 1_4_0); - API_VERSION_HANDLE(1_3_0, 1_4_0); - API_VERSION_HANDLE(1_4_0, 1_4_0); + API_VERSION_HANDLE(1_0_0, 1_4_1); + API_VERSION_HANDLE(1_0_1, 1_4_1); + API_VERSION_HANDLE(1_0_2, 1_4_1); + API_VERSION_HANDLE(1_1_0, 1_4_1); + API_VERSION_HANDLE(1_1_1, 1_4_1); + API_VERSION_HANDLE(1_1_2, 1_4_1); + API_VERSION_HANDLE(1_2_0, 1_4_1); + API_VERSION_HANDLE(1_3_0, 1_4_1); + API_VERSION_HANDLE(1_4_0, 1_4_1); + API_VERSION_HANDLE(1_4_1, 1_4_1); #undef API_VERSION_HANDLE