From 24d0922005ca811d376a0687cb94eab0d1014879 Mon Sep 17 00:00:00 2001 From: baldurk Date: Wed, 23 Feb 2022 10:18:51 +0000 Subject: [PATCH] On GL refuse to capture if any unsupported function is called * For some of these it may be harmless, but it is apparently common for applications to blindly use extenstion functions without checking that the extension is available and this is the only way to make things robust and prevent them from generating potentially broken captures. --- renderdoc/driver/gl/gen_dispatch_table.py | 7 +- renderdoc/driver/gl/gl_dispatch_table_defs.h | 126 +++---------------- renderdoc/driver/gl/gl_driver.cpp | 46 ++++++- renderdoc/driver/gl/gl_driver.h | 3 + renderdoc/driver/gl/gl_hooks.cpp | 8 ++ 5 files changed, 73 insertions(+), 117 deletions(-) diff --git a/renderdoc/driver/gl/gen_dispatch_table.py b/renderdoc/driver/gl/gen_dispatch_table.py index 103689b53..af0f29625 100644 --- a/renderdoc/driver/gl/gen_dispatch_table.py +++ b/renderdoc/driver/gl/gen_dispatch_table.py @@ -259,12 +259,7 @@ template = ''' CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \\ ret HOOK_CC CONCAT(function, _renderdoc_hooked)({argdecl}) \\ {{ \\ - static bool hit = false; \\ - if(hit == false) \\ - {{ \\ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \\ - hit = true; \\ - }} \\ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \\ if(!CONCAT(unsupported_real_, function)) \\ CONCAT(unsupported_real_, function) = \\ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \\ diff --git a/renderdoc/driver/gl/gl_dispatch_table_defs.h b/renderdoc/driver/gl/gl_dispatch_table_defs.h index 7827e88ca..ad8ae18ee 100644 --- a/renderdoc/driver/gl/gl_dispatch_table_defs.h +++ b/renderdoc/driver/gl/gl_dispatch_table_defs.h @@ -6418,12 +6418,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)() \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -6467,12 +6462,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -6516,12 +6506,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -6565,12 +6550,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -6614,12 +6594,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -6663,12 +6638,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -6712,12 +6682,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -6761,12 +6726,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -6810,12 +6770,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -6859,12 +6814,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -6908,12 +6858,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9, t10 p10) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -6957,12 +6902,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9, t10 p10, t11 p11) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -7006,12 +6946,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9, t10 p10, t11 p11, t12 p12) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -7055,12 +6990,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9, t10 p10, t11 p11, t12 p12, t13 p13) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -7104,12 +7034,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9, t10 p10, t11 p11, t12 p12, t13 p13, t14 p14) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -7153,12 +7078,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9, t10 p10, t11 p11, t12 p12, t13 p13, t14 p14, t15 p15) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -7202,12 +7122,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9, t10 p10, t11 p11, t12 p12, t13 p13, t14 p14, t15 p15, t16 p16) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ @@ -7251,12 +7166,7 @@ CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \ ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9, t10 p10, t11 p11, t12 p12, t13 p13, t14 p14, t15 p15, t16 p16, t17 p17) \ { \ - static bool hit = false; \ - if(hit == false) \ - { \ - RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \ - hit = true; \ - } \ + glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \ if(!CONCAT(unsupported_real_, function)) \ CONCAT(unsupported_real_, function) = \ (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \ diff --git a/renderdoc/driver/gl/gl_driver.cpp b/renderdoc/driver/gl/gl_driver.cpp index 29afd9497..e71dfbc8a 100644 --- a/renderdoc/driver/gl/gl_driver.cpp +++ b/renderdoc/driver/gl/gl_driver.cpp @@ -1001,6 +1001,25 @@ GLResourceRecord *WrappedOpenGL::GetContextRecord() } } +void WrappedOpenGL::UseUnusedSupportedFunction(const char *name) +{ + // if this is the first time an unused function is called, remove all frame capturers immediately + if(m_UnsupportedFunctions.empty()) + { + for(auto it = m_ContextData.begin(); it != m_ContextData.end(); ++it) + { + if(it->second.Modern()) + { + RenderDoc::Inst().RemoveDeviceFrameCapturer(it->second.ctx); + for(auto wnd : it->second.windows) + it->second.UnassociateWindow(this, wnd.first); + } + } + } + + m_UnsupportedFunctions.insert(name); +} + void WrappedOpenGL::CheckImplicitThread() { void *ctx = GetCtx().ctx; @@ -1169,7 +1188,9 @@ void WrappedOpenGL::CreateContext(GLWindowingData winData, void *shareContext, RDCLOG("Reusing old sharegroup %p", ctxdata.shareGroup); } - if(attribsCreate) + // if the context was created with modern attribs create (whether or not it's explicitly core), + // and no unsupported functions have been used, we can capture from this context + if(attribsCreate && !m_UnsupportedFunctions.empty()) RenderDoc::Inst().AddDeviceFrameCapturer(ctxdata.ctx, this); // re-configure callstack capture, since WrappedOpenGL constructor may run too early @@ -1974,7 +1995,9 @@ void WrappedOpenGL::SwapBuffers(WindowingSystem winSystem, void *windowHandle) if(it->first != ctxdata.ctx) it->second.UnassociateWindow(this, windowHandle); - ctxdata.AssociateWindow(this, winSystem, windowHandle); + // only associate windows if no unsupported functions have been used + if(m_UnsupportedFunctions.empty()) + ctxdata.AssociateWindow(this, winSystem, windowHandle); } // we used to do this here so it was as late as possible to avoid creating objects on contexts @@ -2016,7 +2039,8 @@ void WrappedOpenGL::SwapBuffers(WindowingSystem winSystem, void *windowHandle) if(overlay & eRENDERDOC_Overlay_Enabled) { int flags = activeWindow ? RenderDoc::eOverlay_ActiveWindow : 0; - if(ctxdata.Legacy()) + // capturing is disabled if unsupported functions have been used, or this context is legacy + if(ctxdata.Legacy() || !m_UnsupportedFunctions.empty()) flags |= RenderDoc::eOverlay_CaptureDisabled; rdcstr overlayText = RenderDoc::Inst().GetOverlayText(GetDriverType(), m_FrameCounter, flags); @@ -2033,6 +2057,22 @@ void WrappedOpenGL::SwapBuffers(WindowingSystem winSystem, void *windowHandle) "supported.\n"; } + // print the unsupported functions (up to a handful) to show + if(!m_UnsupportedFunctions.empty()) + { + overlayText += "Captures disabled.\nUnsupported function used:\n"; + size_t i = 0; + for(const char *func : m_UnsupportedFunctions) + { + i++; + if(i > 4) + break; + overlayText += StringFormat::Fmt(" - %s\n", func); + } + if(m_UnsupportedFunctions.size() > i) + overlayText += " - ...\n"; + } + if(activeWindow && m_FailedFrame > 0) { const char *reasonString = "Unknown reason"; diff --git a/renderdoc/driver/gl/gl_driver.h b/renderdoc/driver/gl/gl_driver.h index 94881fefb..4ab275b04 100644 --- a/renderdoc/driver/gl/gl_driver.h +++ b/renderdoc/driver/gl/gl_driver.h @@ -194,6 +194,8 @@ private: std::set m_AcceptedCtx; + std::set m_UnsupportedFunctions; + rdcarray> m_BufferResizes; public: @@ -619,6 +621,7 @@ public: ContextPair &GetCtx(); GLResourceRecord *GetContextRecord(); + void UseUnusedSupportedFunction(const char *name); void CheckImplicitThread(); void CreateTextureImage(GLuint tex, GLenum internalFormat, GLenum initFormatHint, diff --git a/renderdoc/driver/gl/gl_hooks.cpp b/renderdoc/driver/gl/gl_hooks.cpp index 178a9040d..8ea3e640f 100644 --- a/renderdoc/driver/gl/gl_hooks.cpp +++ b/renderdoc/driver/gl/gl_hooks.cpp @@ -55,6 +55,7 @@ public: void RegisterHooks(); + void UseUnusedSupportedFunction(const char *name); void *GetUnsupportedFunction(const char *name); void *handle = NULL; @@ -215,6 +216,13 @@ void *HookedGetProcAddress(const char *func, void *realFunc) return realFunc; } +void GLHook::UseUnusedSupportedFunction(const char *name) +{ + SCOPED_LOCK(glLock); + if(glhook.driver) + glhook.driver->UseUnusedSupportedFunction(name); +} + void *GLHook::GetUnsupportedFunction(const char *name) { #if ENABLED(RDOC_APPLE)