On GL refuse to capture if any unsupported function is called

* For some of these it may be harmless, but it is apparently common for
  applications to blindly use extenstion functions without checking that the
  extension is available and this is the only way to make things robust and
  prevent them from generating potentially broken captures.

renderdoc/driver/gl/gen_dispatch_table.py

@@ -259,12 +259,7 @@ template = '''
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \\
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)({argdecl}) \\
   {{ \\
-    static bool hit = false; \\
-    if(hit == false) \\
-    {{ \\
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \\
-      hit = true; \\
-    }} \\
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \\
     if(!CONCAT(unsupported_real_, function)) \\
       CONCAT(unsupported_real_, function) = \\
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \\

renderdoc/driver/gl/gl_dispatch_table_defs.h

@@ -6418,12 +6418,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)() \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -6467,12 +6462,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -6516,12 +6506,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -6565,12 +6550,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -6614,12 +6594,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -6663,12 +6638,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -6712,12 +6682,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -6761,12 +6726,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -6810,12 +6770,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -6859,12 +6814,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -6908,12 +6858,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9, t10 p10) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -6957,12 +6902,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9, t10 p10, t11 p11) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -7006,12 +6946,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9, t10 p10, t11 p11, t12 p12) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -7055,12 +6990,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9, t10 p10, t11 p11, t12 p12, t13 p13) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -7104,12 +7034,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9, t10 p10, t11 p11, t12 p12, t13 p13, t14 p14) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -7153,12 +7078,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9, t10 p10, t11 p11, t12 p12, t13 p13, t14 p14, t15 p15) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -7202,12 +7122,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9, t10 p10, t11 p11, t12 p12, t13 p13, t14 p14, t15 p15, t16 p16) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \
@@ -7251,12 +7166,7 @@
   CONCAT(function, _hooktype) CONCAT(unsupported_real_, function) = NULL; \
   ret HOOK_CC CONCAT(function, _renderdoc_hooked)(t1 p1, t2 p2, t3 p3, t4 p4, t5 p5, t6 p6, t7 p7, t8 p8, t9 p9, t10 p10, t11 p11, t12 p12, t13 p13, t14 p14, t15 p15, t16 p16, t17 p17) \
   { \
-    static bool hit = false; \
-    if(hit == false) \
-    { \
-      RDCERR("Function " STRINGIZE(function) " not supported - capture may be broken"); \
-      hit = true; \
-    } \
+    glhook.UseUnusedSupportedFunction(STRINGIZE(function)); \
     if(!CONCAT(unsupported_real_, function)) \
       CONCAT(unsupported_real_, function) = \
           (CONCAT(function, _hooktype))glhook.GetUnsupportedFunction(STRINGIZE(function)); \

renderdoc/driver/gl/gl_driver.cpp

@@ -1001,6 +1001,25 @@ GLResourceRecord *WrappedOpenGL::GetContextRecord()
   }
 }
 
+void WrappedOpenGL::UseUnusedSupportedFunction(const char *name)
+{
+  // if this is the first time an unused function is called, remove all frame capturers immediately
+  if(m_UnsupportedFunctions.empty())
+  {
+    for(auto it = m_ContextData.begin(); it != m_ContextData.end(); ++it)
+    {
+      if(it->second.Modern())
+      {
+        RenderDoc::Inst().RemoveDeviceFrameCapturer(it->second.ctx);
+        for(auto wnd : it->second.windows)
+          it->second.UnassociateWindow(this, wnd.first);
+      }
+    }
+  }
+
+  m_UnsupportedFunctions.insert(name);
+}
+
 void WrappedOpenGL::CheckImplicitThread()
 {
   void *ctx = GetCtx().ctx;
@@ -1169,7 +1188,9 @@ void WrappedOpenGL::CreateContext(GLWindowingData winData, void *shareContext,
     RDCLOG("Reusing old sharegroup %p", ctxdata.shareGroup);
   }
 
-  if(attribsCreate)
+  // if the context was created with modern attribs create (whether or not it's explicitly core),
+  // and no unsupported functions have been used, we can capture from this context
+  if(attribsCreate && !m_UnsupportedFunctions.empty())
     RenderDoc::Inst().AddDeviceFrameCapturer(ctxdata.ctx, this);
 
   // re-configure callstack capture, since WrappedOpenGL constructor may run too early
@@ -1974,7 +1995,9 @@ void WrappedOpenGL::SwapBuffers(WindowingSystem winSystem, void *windowHandle)
       if(it->first != ctxdata.ctx)
         it->second.UnassociateWindow(this, windowHandle);
 
-    ctxdata.AssociateWindow(this, winSystem, windowHandle);
+    // only associate windows if no unsupported functions have been used
+    if(m_UnsupportedFunctions.empty())
+      ctxdata.AssociateWindow(this, winSystem, windowHandle);
   }
 
   // we used to do this here so it was as late as possible to avoid creating objects on contexts
@@ -2016,7 +2039,8 @@ void WrappedOpenGL::SwapBuffers(WindowingSystem winSystem, void *windowHandle)
     if(overlay & eRENDERDOC_Overlay_Enabled)
     {
       int flags = activeWindow ? RenderDoc::eOverlay_ActiveWindow : 0;
-      if(ctxdata.Legacy())
+      // capturing is disabled if unsupported functions have been used, or this context is legacy
+      if(ctxdata.Legacy() || !m_UnsupportedFunctions.empty())
         flags |= RenderDoc::eOverlay_CaptureDisabled;
       rdcstr overlayText = RenderDoc::Inst().GetOverlayText(GetDriverType(), m_FrameCounter, flags);
 
@@ -2033,6 +2057,22 @@ void WrappedOpenGL::SwapBuffers(WindowingSystem winSystem, void *windowHandle)
             "supported.\n";
       }
 
+      // print the unsupported functions (up to a handful) to show
+      if(!m_UnsupportedFunctions.empty())
+      {
+        overlayText += "Captures disabled.\nUnsupported function used:\n";
+        size_t i = 0;
+        for(const char *func : m_UnsupportedFunctions)
+        {
+          i++;
+          if(i > 4)
+            break;
+          overlayText += StringFormat::Fmt(" - %s\n", func);
+        }
+        if(m_UnsupportedFunctions.size() > i)
+          overlayText += " - ...\n";
+      }
+
       if(activeWindow && m_FailedFrame > 0)
       {
         const char *reasonString = "Unknown reason";

renderdoc/driver/gl/gl_driver.h

@@ -194,6 +194,8 @@ private:
 
   std::set<void *> m_AcceptedCtx;
 
+  std::set<const char *> m_UnsupportedFunctions;
+
   rdcarray<rdcpair<GLResourceRecord *, Chunk *>> m_BufferResizes;
 
 public:
@@ -619,6 +621,7 @@ public:
   ContextPair &GetCtx();
   GLResourceRecord *GetContextRecord();
 
+  void UseUnusedSupportedFunction(const char *name);
   void CheckImplicitThread();
 
   void CreateTextureImage(GLuint tex, GLenum internalFormat, GLenum initFormatHint,

renderdoc/driver/gl/gl_hooks.cpp

@@ -55,6 +55,7 @@ public:
 
   void RegisterHooks();
 
+  void UseUnusedSupportedFunction(const char *name);
   void *GetUnsupportedFunction(const char *name);
 
   void *handle = NULL;
@@ -215,6 +216,13 @@ void *HookedGetProcAddress(const char *func, void *realFunc)
   return realFunc;
 }
 
+void GLHook::UseUnusedSupportedFunction(const char *name)
+{
+  SCOPED_LOCK(glLock);
+  if(glhook.driver)
+    glhook.driver->UseUnusedSupportedFunction(name);
+}
+
 void *GLHook::GetUnsupportedFunction(const char *name)
 {
 #if ENABLED(RDOC_APPLE)