Remove the ability to enable/disable hooks

* This wasn't very well supported after D3D11 anyway, and modern tools
  (VSGD/PIX/etc) don't respect D3DPERF_SetOptions which was the only route that
  actually needed to globally enable/disable hooks.
* D3D11CreateDevices's flag still works to prevent hooking.

renderdoc/driver/d3d11/d3d11_hooks.cpp

@@ -107,7 +107,6 @@ public:
   {
     LibraryHooks::GetInstance().RegisterHook(DLL_NAME, this);
     m_HasHooks = false;
-    m_EnabledHooks = true;
     m_InsideCreate = false;
   }
 
@@ -158,14 +157,11 @@ public:
       return false;
 
     m_HasHooks = true;
-    m_EnabledHooks = true;
 
     return true;
   }
 
-  void EnableHooks(const char *libName, bool enable) { m_EnabledHooks = enable; }
   void OptionsUpdated(const char *libName) {}
-  bool UseHooks() { return (d3d11hooks.m_HasHooks && d3d11hooks.m_EnabledHooks); }
   static HRESULT CreateWrappedDeviceAndSwapChain(
       __in_opt IDXGIAdapter *pAdapter, D3D_DRIVER_TYPE DriverType, HMODULE Software, UINT Flags,
       __in_ecount_opt(FeatureLevels) CONST D3D_FEATURE_LEVEL *pFeatureLevels, UINT FeatureLevels,
@@ -182,7 +178,6 @@ private:
   static D3D11Hook d3d11hooks;
 
   bool m_HasHooks;
-  bool m_EnabledHooks;
 
   Hook<PFN_D3D11_CREATE_DEVICE_AND_SWAP_CHAIN> CreateDeviceAndSwapChain;
   Hook<PFN_D3D11_CREATE_DEVICE> CreateDevice;
@@ -247,16 +242,13 @@ private:
       RDCDEBUG("In replay app");
     }
 
-    if(m_EnabledHooks)
+    if(!reading && RenderDoc::Inst().GetCaptureOptions().apiValidation)
     {
-      if(!reading && RenderDoc::Inst().GetCaptureOptions().apiValidation)
-      {
-        Flags |= D3D11_CREATE_DEVICE_DEBUG;
-      }
-      else
-      {
-        Flags &= ~D3D11_CREATE_DEVICE_DEBUG;
-      }
+      Flags |= D3D11_CREATE_DEVICE_DEBUG;
+    }
+    else
+    {
+      Flags &= ~D3D11_CREATE_DEVICE_DEBUG;
     }
 
     DXGI_SWAP_CHAIN_DESC swapDesc;
@@ -268,7 +260,7 @@ private:
       pUsedSwapDesc = &swapDesc;
     }
 
-    if(pUsedSwapDesc && m_EnabledHooks && !RenderDoc::Inst().GetCaptureOptions().allowFullscreen)
+    if(pUsedSwapDesc && !RenderDoc::Inst().GetCaptureOptions().allowFullscreen)
     {
       pUsedSwapDesc->Windowed = TRUE;
     }
@@ -316,9 +308,9 @@ private:
 
     if(suppress && !reading)
     {
-      RDCDEBUG("Application requested not to be hooked.");
+      RDCLOG("Application requested not to be hooked.");
     }
-    else if(SUCCEEDED(ret) && m_EnabledHooks && ppDevice)
+    else if(SUCCEEDED(ret) && ppDevice)
     {
       WrapDevice(ppDevice, DriverType, Flags, SDKVersion, FeatureLevels, pFeatureLevels,
                  ppImmediateContext, ppSwapChain, pSwapChainDesc);
@@ -430,16 +422,13 @@ private:
       RDCDEBUG("In replay app");
     }
 
-    if(d3d11hooks.m_EnabledHooks)
+    if(!reading && RenderDoc::Inst().GetCaptureOptions().apiValidation)
     {
-      if(!reading && RenderDoc::Inst().GetCaptureOptions().apiValidation)
-      {
-        Flags |= D3D11_CREATE_DEVICE_DEBUG;
-      }
-      else
-      {
-        Flags &= ~D3D11_CREATE_DEVICE_DEBUG;
-      }
+      Flags |= D3D11_CREATE_DEVICE_DEBUG;
+    }
+    else
+    {
+      Flags &= ~D3D11_CREATE_DEVICE_DEBUG;
     }
 
     RDCDEBUG("Calling real createdevice...");
@@ -458,7 +447,7 @@ private:
     {
       RDCDEBUG("Application requested not to be hooked.");
     }
-    else if(SUCCEEDED(ret) && d3d11hooks.m_EnabledHooks && ppDevice)
+    else if(SUCCEEDED(ret) && ppDevice)
     {
       d3d11hooks.WrapDevice(ppDevice, DriverType, Flags, SDKVersion, FeatureLevels, pFeatureLevels,
                             ppImmediateContext, NULL, NULL);
@@ -491,16 +480,13 @@ private:
       RDCDEBUG("In replay app");
     }
 
-    if(d3d11hooks.m_EnabledHooks)
+    if(!reading && RenderDoc::Inst().GetCaptureOptions().apiValidation)
     {
-      if(!reading && RenderDoc::Inst().GetCaptureOptions().apiValidation)
-      {
-        Flags |= D3D11_CREATE_DEVICE_DEBUG;
-      }
-      else
-      {
-        Flags &= ~D3D11_CREATE_DEVICE_DEBUG;
-      }
+      Flags |= D3D11_CREATE_DEVICE_DEBUG;
+    }
+    else
+    {
+      Flags &= ~D3D11_CREATE_DEVICE_DEBUG;
     }
 
     DXGI_SWAP_CHAIN_DESC swapDesc;
@@ -512,8 +498,7 @@ private:
       pUsedSwapDesc = &swapDesc;
     }
 
-    if(pUsedSwapDesc && d3d11hooks.m_EnabledHooks &&
-       !RenderDoc::Inst().GetCaptureOptions().allowFullscreen)
+    if(pUsedSwapDesc && !RenderDoc::Inst().GetCaptureOptions().allowFullscreen)
     {
       pUsedSwapDesc->Windowed = TRUE;
     }
@@ -534,7 +519,7 @@ private:
     {
       RDCDEBUG("Application requested not to be hooked.");
     }
-    else if(SUCCEEDED(ret) && d3d11hooks.m_EnabledHooks && ppDevice)
+    else if(SUCCEEDED(ret) && ppDevice)
     {
       d3d11hooks.WrapDevice(ppDevice, DriverType, Flags, SDKVersion, FeatureLevels, pFeatureLevels,
                             ppImmediateContext, ppSwapChain, pSwapChainDesc);

renderdoc/driver/d3d12/d3d12_hooks.cpp

@@ -92,7 +92,6 @@ public:
   {
     LibraryHooks::GetInstance().RegisterHook(DLL_NAME, this);
     m_HasHooks = false;
-    m_EnabledHooks = true;
     m_InsideCreate = false;
   }
 
@@ -119,14 +118,11 @@ public:
       return false;
 
     m_HasHooks = true;
-    m_EnabledHooks = true;
 
     return true;
   }
 
-  void EnableHooks(const char *libName, bool enable) { m_EnabledHooks = enable; }
   void OptionsUpdated(const char *libName) {}
-  bool UseHooks() { return (d3d12hooks.m_HasHooks && d3d12hooks.m_EnabledHooks); }
   static HRESULT CreateWrappedDevice(IUnknown *pAdapter, D3D_FEATURE_LEVEL MinimumFeatureLevel,
                                      REFIID riid, void **ppDevice)
   {
@@ -137,7 +133,6 @@ private:
   static D3D12Hook d3d12hooks;
 
   bool m_HasHooks;
-  bool m_EnabledHooks;
 
   Hook<PFN_D3D12_GET_DEBUG_INTERFACE> GetDebugInterface;
   Hook<PFN_D3D12_CREATE_DEVICE> CreateDevice;
@@ -201,7 +196,7 @@ private:
 #if ENABLED(RDOC_DEVEL)
         RenderDoc::Inst().IsReplayApp() ||
 #endif
-        (m_EnabledHooks && !reading && RenderDoc::Inst().GetCaptureOptions().apiValidation);
+        (!reading && RenderDoc::Inst().GetCaptureOptions().apiValidation);
 
     if(EnableDebugLayer)
     {
@@ -275,7 +270,7 @@ private:
 
     RDCDEBUG("Called real createdevice... HRESULT: %s", ToStr(ret).c_str());
 
-    if(SUCCEEDED(ret) && m_EnabledHooks && ppDevice)
+    if(SUCCEEDED(ret) && ppDevice)
     {
       RDCDEBUG("succeeded and hooking.");
 

renderdoc/driver/d3d8/d3d8_hooks.cpp

@@ -37,7 +37,6 @@ public:
   {
     LibraryHooks::GetInstance().RegisterHook(DLL_NAME, this);
     m_HasHooks = false;
-    m_EnabledHooks = true;
   }
 
   bool CreateHooks(const char *libName)
@@ -50,18 +49,15 @@ public:
       return false;
 
     m_HasHooks = true;
-    m_EnabledHooks = true;
 
     return true;
   }
 
-  void EnableHooks(const char *libName, bool enable) { m_EnabledHooks = enable; }
   void OptionsUpdated(const char *libName) {}
 private:
   static D3D8Hook d3d8hooks;
 
   bool m_HasHooks;
-  bool m_EnabledHooks;
 
   Hook<PFN_D3D8_CREATE> Create8;
 

renderdoc/driver/d3d9/d3d9_hooks.cpp

@@ -46,7 +46,6 @@ public:
   {
     LibraryHooks::GetInstance().RegisterHook(DLL_NAME, this);
     m_HasHooks = false;
-    m_EnabledHooks = true;
   }
 
   bool CreateHooks(const char *libName)
@@ -65,18 +64,15 @@ public:
       return false;
 
     m_HasHooks = true;
-    m_EnabledHooks = true;
 
     return true;
   }
 
-  void EnableHooks(const char *libName, bool enable) { m_EnabledHooks = enable; }
   void OptionsUpdated(const char *libName) {}
 private:
   static D3D9Hook d3d9hooks;
 
   bool m_HasHooks;
-  bool m_EnabledHooks;
 
   // D3DPERF api
   Hook<PFN_BEGIN_EVENT> PERF_BeginEvent;
@@ -114,26 +110,10 @@ private:
   static void WINAPI PERF_SetOptions_hook(DWORD dwOptions)
   {
     if(dwOptions & 1)
-    {
-      RDCDEBUG("Application requested not to be hooked.");
-      LibraryHooks::GetInstance().EnableHooks(false);
-    }
-    else
-    {
-      LibraryHooks::GetInstance().EnableHooks(true);
-    }
-  }
-
-  static DWORD WINAPI PERF_GetStatus_hook()
-  {
-    if(d3d9hooks.m_HasHooks && d3d9hooks.m_EnabledHooks)
-    {
-      return 1;
-    }
-
-    return 0;
+      RDCLOG("Application requested not to be hooked via D3DPERF_SetOptions: no longer supported.");
   }
 
+  static DWORD WINAPI PERF_GetStatus_hook() { return 1; }
   static IDirect3D9 *WINAPI Create9_hook(UINT SDKVersion)
   {
     RDCLOG("App creating d3d9 %x", SDKVersion);

renderdoc/driver/dxgi/dxgi_hooks.cpp

@@ -77,7 +77,6 @@ public:
   DXGIHook()
   {
     LibraryHooks::GetInstance().RegisterHook(DLL_NAME, this);
-    m_EnabledHooks = true;
     m_HasHooks = false;
   }
 
@@ -96,14 +95,11 @@ public:
       return false;
 
     m_HasHooks = true;
-    m_EnabledHooks = true;
 
     return true;
   }
 
-  void EnableHooks(const char *libName, bool enable) { m_EnabledHooks = enable; }
   void OptionsUpdated(const char *libName) {}
-  bool UseHooks() { return (dxgihooks.m_HasHooks && dxgihooks.m_EnabledHooks); }
   static HRESULT CreateWrappedFactory1(REFIID riid, void **ppFactory)
   {
     if(dxgihooks.m_HasHooks)
@@ -172,7 +168,6 @@ private:
   static DXGIHook dxgihooks;
 
   bool m_HasHooks;
-  bool m_EnabledHooks;
 
   RenderDocAnalysis m_RenderDocAnalysis;
 
@@ -188,7 +183,7 @@ private:
       *ppFactory = NULL;
     HRESULT ret = dxgihooks.CreateDXGIFactory()(riid, ppFactory);
 
-    if(SUCCEEDED(ret) && dxgihooks.m_EnabledHooks)
+    if(SUCCEEDED(ret))
       RefCountDXGIObject::HandleWrap(riid, ppFactory);
 
     return ret;
@@ -200,7 +195,7 @@ private:
       *ppFactory = NULL;
     HRESULT ret = dxgihooks.CreateDXGIFactory1()(riid, ppFactory);
 
-    if(SUCCEEDED(ret) && dxgihooks.m_EnabledHooks)
+    if(SUCCEEDED(ret))
       RefCountDXGIObject::HandleWrap(riid, ppFactory);
 
     return ret;
@@ -212,7 +207,7 @@ private:
       *ppFactory = NULL;
     HRESULT ret = dxgihooks.CreateDXGIFactory2()(Flags, riid, ppFactory);
 
-    if(SUCCEEDED(ret) && dxgihooks.m_EnabledHooks)
+    if(SUCCEEDED(ret))
       RefCountDXGIObject::HandleWrap(riid, ppFactory);
 
     return ret;

renderdoc/driver/gl/gl_hooks_apple.cpp

@@ -39,7 +39,6 @@ public:
   OpenGLHook() {}
   ~OpenGLHook() {}
   bool CreateHooks(const char *libName) { return false; }
-  void EnableHooks(const char *libName, bool enable) {}
   void OptionsUpdated(const char *libName) {}
   virtual GLWindowingData MakeContext(GLWindowingData share)
   {

renderdoc/driver/gl/gl_hooks_egl.cpp

@@ -43,7 +43,6 @@ public:
 
     m_GLDriver = NULL;
 
-    m_EnabledHooks = true;
     m_PopulatedHooks = false;
   }
   ~EGLHook()
@@ -55,7 +54,6 @@ public:
 
   bool CreateHooks(const char *libName);
 
-  void EnableHooks(const char *libName, bool enable) { m_EnabledHooks = enable; }
   void OptionsUpdated(const char *libName) {}
   void PopulateEGLFunctions()
   {
@@ -179,7 +177,6 @@ public:
 
   bool m_PopulatedHooks;
   bool m_HasHooks;
-  bool m_EnabledHooks;
 
   bool SetupHooks()
   {
@@ -415,9 +412,6 @@ void EGLHook::libHooked(void *realLib)
 
 bool EGLHook::CreateHooks(const char *libName)
 {
-  if(!m_EnabledHooks)
-    return false;
-
   if(libName)
   {
     // register our hooked functions for PLT hooking on android

renderdoc/driver/gl/gl_hooks_linux.cpp

@@ -55,7 +55,6 @@ public:
 
     m_GLDriver = NULL;
 
-    m_EnabledHooks = true;
     m_PopulatedHooks = false;
   }
   ~OpenGLHook()
@@ -67,9 +66,6 @@ public:
 
   bool CreateHooks(const char *libName)
   {
-    if(!m_EnabledHooks)
-      return false;
-
     if(libName)
       PosixHookLibrary("libGL.so", &libHooked);
 
@@ -83,7 +79,6 @@ public:
     return true;
   }
 
-  void EnableHooks(const char *libName, bool enable) { m_EnabledHooks = enable; }
   void OptionsUpdated(const char *libName) {}
   // see callsite in glXSwapBuffers for explanation of why this is necessary
   XID UnwrapGLXWindow(XID id)
@@ -395,7 +390,6 @@ public:
 
   bool m_PopulatedHooks;
   bool m_HasHooks;
-  bool m_EnabledHooks;
 
   bool SetupHooks()
   {

renderdoc/driver/gl/gl_hooks_win32.cpp

@@ -675,7 +675,6 @@ public:
 
     m_HaveContextCreation = false;
 
-    m_EnabledHooks = true;
     m_PopulatedHooks = false;
 
     m_CreatingContext = false;
@@ -685,9 +684,6 @@ public:
   ~OpenGLHook() { delete m_GLDriver; }
   bool CreateHooks(const char *libName)
   {
-    if(!m_EnabledHooks)
-      return false;
-
     bool success = SetupHooks();
 
     if(!success)
@@ -698,7 +694,6 @@ public:
     return true;
   }
 
-  void EnableHooks(const char *libName, bool enable) { m_EnabledHooks = enable; }
   void OptionsUpdated(const char *libName) {}
   static OpenGLHook glhooks;
 
@@ -1370,7 +1365,6 @@ private:
 
   bool m_PopulatedHooks;
   bool m_HasHooks;
-  bool m_EnabledHooks;
 
   set<HGLRC> m_Contexts;
 

renderdoc/driver/vulkan/vk_layer.cpp

@@ -60,15 +60,6 @@ class VulkanHook : LibraryHook
     return true;
   }
 
-  void EnableHooks(const char *libName, bool enable)
-  {
-    // set the env var to 0 to disable the implicit layer
-    Process::RegisterEnvironmentModification(EnvironmentModification(
-        EnvMod::Set, EnvSep::NoSep, "ENABLE_VULKAN_RENDERDOC_CAPTURE", enable ? "1" : "0"));
-
-    Process::ApplyEnvironmentModification();
-  }
-
   void OptionsUpdated(const char *libName)
   {
     if(RenderDoc::Inst().GetCaptureOptions().apiValidation)

renderdoc/hooks/hooks.cpp

@@ -58,14 +58,6 @@ void LibraryHooks::RemoveHooks()
   HOOKS_REMOVE();
 }
 
-void LibraryHooks::EnableHooks(bool enable)
-{
-  RDCDEBUG("%s hooks!", enable ? "Enabling" : "Disabling");
-
-  for(auto it = m_Hooks.begin(); it != m_Hooks.end(); ++it)
-    it->second->EnableHooks(it->first, enable);
-}
-
 void LibraryHooks::OptionsUpdated()
 {
   for(auto it = m_Hooks.begin(); it != m_Hooks.end(); ++it)

renderdoc/hooks/hooks.h

@@ -83,7 +83,6 @@ private:
 struct LibraryHook
 {
   virtual bool CreateHooks(const char *libName) = 0;
-  virtual void EnableHooks(const char *libName, bool enable) = 0;
   virtual void OptionsUpdated(const char *libName) = 0;
 };
 
@@ -98,7 +97,6 @@ public:
   void RegisterHook(const char *libName, LibraryHook *hook);
   void CreateHooks();
   void OptionsUpdated();
-  void EnableHooks(bool enable);
   void RemoveHooks();
 
 private:

renderdoc/os/win32/sys_win32_hooks.cpp

@@ -76,7 +76,6 @@ public:
   SysHook()
   {
     LibraryHooks::GetInstance().RegisterHook(DLL_NAME, this);
-    m_EnabledHooks = true;
     m_HasHooks = false;
     m_WSARefCount = 1;
   }
@@ -138,18 +137,15 @@ public:
       return false;
 
     m_HasHooks = true;
-    m_EnabledHooks = true;
 
     return true;
   }
 
-  void EnableHooks(const char *libName, bool enable) { m_EnabledHooks = enable; }
   void OptionsUpdated(const char *libName) {}
 private:
   static SysHook syshooks;
 
   bool m_HasHooks;
-  bool m_EnabledHooks;
 
   int m_WSARefCount;
   uint64_t m_RecurseSlot = 0;