Handle querying out cubemap face of FBO separate from layer. Refs #341

* In GL although the ARB_dsa extension allows binding of cubemap faces
  as if they were a 2D array (you know, the sane way), it doesn't let
  the queries return the cubemap face to the LAYER query. Instead it
  needs a separate query of CUBE_MAP_FACE.

renderdoc/driver/gl/gl_manager.cpp

@@ -318,14 +318,32 @@ bool GLResourceManager::Prepare_InitialState(GLResource res, byte *blob)
             res.name, data->attachmentNames[i], eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LEVEL, &a.level);
         gl.glGetNamedFramebufferAttachmentParameterivEXT(
             res.name, data->attachmentNames[i], eGL_FRAMEBUFFER_ATTACHMENT_LAYERED, &layered);
-        gl.glGetNamedFramebufferAttachmentParameterivEXT(
-            res.name, data->attachmentNames[i], eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LAYER, &a.layer);
+
+        a.layer = 0;
+        if(layered == 0)
+          gl.glGetNamedFramebufferAttachmentParameterivEXT(
+              res.name, data->attachmentNames[i], eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LAYER, &a.layer);
       }
 
       a.layered = (layered != 0);
       a.renderbuffer = (type == eGL_RENDERBUFFER);
       a.obj = GetID(a.renderbuffer ? RenderbufferRes(res.Context, object)
                                    : TextureRes(res.Context, object));
+
+      if(!a.renderbuffer)
+      {
+        WrappedOpenGL::TextureData &details = m_GL->m_Textures[a.obj];
+
+        if(details.curType == eGL_TEXTURE_CUBE_MAP)
+        {
+          GLenum face;
+          gl.glGetNamedFramebufferAttachmentParameterivEXT(
+              res.name, data->attachmentNames[i], eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_CUBE_MAP_FACE,
+              (GLint *)&face);
+
+          a.layer = CubeTargetIndex(face);
+        }
+      }
     }
 
     for(int i = 0; i < (int)ARRAY_COUNT(data->DrawBuffers); i++)
@@ -1830,10 +1848,39 @@ void GLResourceManager::Apply_InitialState(GLResource live, InitialContentData i
         }
         else
         {
-          if(a.layered && obj)
+          if(!a.layered && obj)
           {
-            gl.glNamedFramebufferTextureLayerEXT(live.name, data->attachmentNames[i], obj, a.level,
-                                                 a.layer);
+            // we use old-style non-DSA for this because binding cubemap faces with EXT_dsa
+            // is completely messed up and broken
+
+            // if obj is a cubemap use face-specific targets
+            WrappedOpenGL::TextureData &details = m_GL->m_Textures[GetLiveID(a.obj)];
+
+            if(details.curType == eGL_TEXTURE_CUBE_MAP)
+            {
+              GLenum faces[] = {
+                  eGL_TEXTURE_CUBE_MAP_POSITIVE_X, eGL_TEXTURE_CUBE_MAP_NEGATIVE_X,
+                  eGL_TEXTURE_CUBE_MAP_POSITIVE_Y, eGL_TEXTURE_CUBE_MAP_NEGATIVE_Y,
+                  eGL_TEXTURE_CUBE_MAP_POSITIVE_Z, eGL_TEXTURE_CUBE_MAP_NEGATIVE_Z,
+              };
+
+              if(a.layer < 6)
+              {
+                gl.glFramebufferTexture2D(eGL_DRAW_FRAMEBUFFER, data->attachmentNames[i],
+                                          faces[a.layer], obj, a.level);
+              }
+              else
+              {
+                RDCWARN("Invalid layer %u used to bind cubemap to framebuffer. Binding POSITIVE_X");
+                gl.glFramebufferTexture2D(eGL_DRAW_FRAMEBUFFER, data->attachmentNames[i], faces[0],
+                                          obj, a.level);
+              }
+            }
+            else
+            {
+              gl.glFramebufferTextureLayer(eGL_DRAW_FRAMEBUFFER, data->attachmentNames[i], obj,
+                                           a.level, a.layer);
+            }
           }
           else
           {

renderdoc/driver/gl/gl_replay.cpp

@@ -1753,20 +1753,10 @@ void GLReplay::SavePipelineState()
           rm->GetID(rbCol[i] ? RenderbufferRes(ctx, curCol[i]) : TextureRes(ctx, curCol[i])));
 
       if(pipe.m_FB.m_DrawFBO.Color[i].Obj != ResourceId() && !rbCol[i])
-      {
-        gl.glGetFramebufferAttachmentParameteriv(
-            eGL_DRAW_FRAMEBUFFER, GLenum(eGL_COLOR_ATTACHMENT0 + i),
-            eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LEVEL, (GLint *)&pipe.m_FB.m_DrawFBO.Color[i].Mip);
-        gl.glGetFramebufferAttachmentParameteriv(eGL_DRAW_FRAMEBUFFER,
-                                                 GLenum(eGL_COLOR_ATTACHMENT0 + i),
-                                                 eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_CUBE_MAP_FACE,
-                                                 (GLint *)&pipe.m_FB.m_DrawFBO.Color[i].Layer);
-        if(pipe.m_FB.m_DrawFBO.Color[i].Layer == 0)
-          gl.glGetFramebufferAttachmentParameteriv(eGL_DRAW_FRAMEBUFFER,
-                                                   GLenum(eGL_COLOR_ATTACHMENT0 + i),
-                                                   eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LAYER,
-                                                   (GLint *)&pipe.m_FB.m_DrawFBO.Color[i].Layer);
-      }
+        GetFramebufferMipAndLayer(gl.GetHookset(), eGL_DRAW_FRAMEBUFFER,
+                                  GLenum(eGL_COLOR_ATTACHMENT0 + i),
+                                  (GLint *)&pipe.m_FB.m_DrawFBO.Color[i].Mip,
+                                  (GLint *)&pipe.m_FB.m_DrawFBO.Color[i].Layer);
     }
 
     pipe.m_FB.m_DrawFBO.Depth.Obj = rm->GetOriginalID(
@@ -1775,32 +1765,14 @@ void GLReplay::SavePipelineState()
         rm->GetID(rbStencil ? RenderbufferRes(ctx, curStencil) : TextureRes(ctx, curStencil)));
 
     if(pipe.m_FB.m_DrawFBO.Depth.Obj != ResourceId() && !rbDepth)
-    {
-      gl.glGetFramebufferAttachmentParameteriv(eGL_DRAW_FRAMEBUFFER, eGL_DEPTH_ATTACHMENT,
-                                               eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LEVEL,
-                                               (GLint *)&pipe.m_FB.m_DrawFBO.Depth.Mip);
-      gl.glGetFramebufferAttachmentParameteriv(eGL_DRAW_FRAMEBUFFER, eGL_DEPTH_ATTACHMENT,
-                                               eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_CUBE_MAP_FACE,
-                                               (GLint *)&pipe.m_FB.m_DrawFBO.Depth.Layer);
-      if(pipe.m_FB.m_DrawFBO.Depth.Layer == 0)
-        gl.glGetFramebufferAttachmentParameteriv(eGL_DRAW_FRAMEBUFFER, eGL_DEPTH_ATTACHMENT,
-                                                 eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LAYER,
-                                                 (GLint *)&pipe.m_FB.m_DrawFBO.Depth.Layer);
-    }
+      GetFramebufferMipAndLayer(gl.GetHookset(), eGL_DRAW_FRAMEBUFFER, eGL_DEPTH_ATTACHMENT,
+                                (GLint *)&pipe.m_FB.m_DrawFBO.Depth.Mip,
+                                (GLint *)&pipe.m_FB.m_DrawFBO.Depth.Layer);
 
     if(pipe.m_FB.m_DrawFBO.Stencil.Obj != ResourceId() && !rbStencil)
-    {
-      gl.glGetFramebufferAttachmentParameteriv(eGL_DRAW_FRAMEBUFFER, eGL_STENCIL_ATTACHMENT,
-                                               eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LEVEL,
-                                               (GLint *)&pipe.m_FB.m_DrawFBO.Stencil.Mip);
-      gl.glGetFramebufferAttachmentParameteriv(eGL_DRAW_FRAMEBUFFER, eGL_STENCIL_ATTACHMENT,
-                                               eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_CUBE_MAP_FACE,
-                                               (GLint *)&pipe.m_FB.m_DrawFBO.Stencil.Layer);
-      if(pipe.m_FB.m_DrawFBO.Stencil.Layer == 0)
-        gl.glGetFramebufferAttachmentParameteriv(eGL_DRAW_FRAMEBUFFER, eGL_STENCIL_ATTACHMENT,
-                                                 eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LAYER,
-                                                 (GLint *)&pipe.m_FB.m_DrawFBO.Stencil.Layer);
-    }
+      GetFramebufferMipAndLayer(gl.GetHookset(), eGL_DRAW_FRAMEBUFFER, eGL_STENCIL_ATTACHMENT,
+                                (GLint *)&pipe.m_FB.m_DrawFBO.Stencil.Mip,
+                                (GLint *)&pipe.m_FB.m_DrawFBO.Stencil.Layer);
 
     create_array_uninit(pipe.m_FB.m_DrawFBO.DrawBuffers, numCols);
     for(GLint i = 0; i < numCols; i++)
@@ -1853,20 +1825,10 @@ void GLReplay::SavePipelineState()
           rm->GetID(rbCol[i] ? RenderbufferRes(ctx, curCol[i]) : TextureRes(ctx, curCol[i])));
 
       if(pipe.m_FB.m_ReadFBO.Color[i].Obj != ResourceId() && !rbCol[i])
-      {
-        gl.glGetFramebufferAttachmentParameteriv(
-            eGL_READ_FRAMEBUFFER, GLenum(eGL_COLOR_ATTACHMENT0 + i),
-            eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LEVEL, (GLint *)&pipe.m_FB.m_ReadFBO.Color[i].Mip);
-        gl.glGetFramebufferAttachmentParameteriv(eGL_READ_FRAMEBUFFER,
-                                                 GLenum(eGL_COLOR_ATTACHMENT0 + i),
-                                                 eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_CUBE_MAP_FACE,
-                                                 (GLint *)&pipe.m_FB.m_ReadFBO.Color[i].Layer);
-        if(pipe.m_FB.m_ReadFBO.Color[i].Layer == 0)
-          gl.glGetFramebufferAttachmentParameteriv(eGL_READ_FRAMEBUFFER,
-                                                   GLenum(eGL_COLOR_ATTACHMENT0 + i),
-                                                   eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LAYER,
-                                                   (GLint *)&pipe.m_FB.m_ReadFBO.Color[i].Layer);
-      }
+        GetFramebufferMipAndLayer(gl.GetHookset(), eGL_READ_FRAMEBUFFER,
+                                  GLenum(eGL_COLOR_ATTACHMENT0 + i),
+                                  (GLint *)&pipe.m_FB.m_ReadFBO.Color[i].Mip,
+                                  (GLint *)&pipe.m_FB.m_ReadFBO.Color[i].Layer);
     }
 
     pipe.m_FB.m_ReadFBO.Depth.Obj = rm->GetOriginalID(
@@ -1875,32 +1837,14 @@ void GLReplay::SavePipelineState()
         rm->GetID(rbStencil ? RenderbufferRes(ctx, curStencil) : TextureRes(ctx, curStencil)));
 
     if(pipe.m_FB.m_ReadFBO.Depth.Obj != ResourceId() && !rbDepth)
-    {
-      gl.glGetFramebufferAttachmentParameteriv(eGL_READ_FRAMEBUFFER, eGL_DEPTH_ATTACHMENT,
-                                               eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LEVEL,
-                                               (GLint *)&pipe.m_FB.m_ReadFBO.Depth.Mip);
-      gl.glGetFramebufferAttachmentParameteriv(eGL_READ_FRAMEBUFFER, eGL_DEPTH_ATTACHMENT,
-                                               eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_CUBE_MAP_FACE,
-                                               (GLint *)&pipe.m_FB.m_ReadFBO.Depth.Layer);
-      if(pipe.m_FB.m_ReadFBO.Depth.Layer == 0)
-        gl.glGetFramebufferAttachmentParameteriv(eGL_READ_FRAMEBUFFER, eGL_DEPTH_ATTACHMENT,
-                                                 eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LAYER,
-                                                 (GLint *)&pipe.m_FB.m_ReadFBO.Depth.Layer);
-    }
+      GetFramebufferMipAndLayer(gl.GetHookset(), eGL_READ_FRAMEBUFFER, eGL_DEPTH_ATTACHMENT,
+                                (GLint *)&pipe.m_FB.m_ReadFBO.Depth.Mip,
+                                (GLint *)&pipe.m_FB.m_ReadFBO.Depth.Layer);
 
     if(pipe.m_FB.m_ReadFBO.Stencil.Obj != ResourceId() && !rbStencil)
-    {
-      gl.glGetFramebufferAttachmentParameteriv(eGL_READ_FRAMEBUFFER, eGL_STENCIL_ATTACHMENT,
-                                               eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LEVEL,
-                                               (GLint *)&pipe.m_FB.m_ReadFBO.Stencil.Mip);
-      gl.glGetFramebufferAttachmentParameteriv(eGL_READ_FRAMEBUFFER, eGL_STENCIL_ATTACHMENT,
-                                               eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_CUBE_MAP_FACE,
-                                               (GLint *)&pipe.m_FB.m_ReadFBO.Stencil.Layer);
-      if(pipe.m_FB.m_ReadFBO.Stencil.Layer == 0)
-        gl.glGetFramebufferAttachmentParameteriv(eGL_READ_FRAMEBUFFER, eGL_STENCIL_ATTACHMENT,
-                                                 eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LAYER,
-                                                 (GLint *)&pipe.m_FB.m_ReadFBO.Stencil.Layer);
-    }
+      GetFramebufferMipAndLayer(gl.GetHookset(), eGL_READ_FRAMEBUFFER, eGL_STENCIL_ATTACHMENT,
+                                (GLint *)&pipe.m_FB.m_ReadFBO.Stencil.Mip,
+                                (GLint *)&pipe.m_FB.m_ReadFBO.Stencil.Layer);
 
     create_array_uninit(pipe.m_FB.m_ReadFBO.DrawBuffers, numCols);
     for(GLint i = 0; i < numCols; i++)

renderdoc/driver/gl/gl_resources.cpp

@@ -610,6 +610,27 @@ GLenum GetSizedFormat(const GLHookSet &gl, GLenum target, GLenum internalFormat)
   return internalFormat;
 }
 
+void GetFramebufferMipAndLayer(const GLHookSet &gl, GLenum framebuffer, GLenum attachment,
+                               GLint *mip, GLint *layer)
+{
+  gl.glGetFramebufferAttachmentParameteriv(framebuffer, attachment,
+                                           eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LEVEL, mip);
+
+  GLenum face = eGL_NONE;
+  gl.glGetFramebufferAttachmentParameteriv(
+      framebuffer, attachment, eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_CUBE_MAP_FACE, (GLint *)&face);
+
+  if(face == 0)
+  {
+    gl.glGetFramebufferAttachmentParameteriv(framebuffer, attachment,
+                                             eGL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LAYER, layer);
+  }
+  else
+  {
+    *layer = CubeTargetIndex(face);
+  }
+}
+
 bool EmulateLuminanceFormat(const GLHookSet &gl, GLuint tex, GLenum target, GLenum &internalFormat,
                             GLenum &dataFormat)
 {
@@ -919,6 +940,22 @@ GLenum BufferBinding(GLenum target)
   return eGL_NONE;
 }
 
+GLint CubeTargetIndex(GLenum face)
+{
+  switch(face)
+  {
+    case eGL_TEXTURE_CUBE_MAP_POSITIVE_X: return 0;
+    case eGL_TEXTURE_CUBE_MAP_NEGATIVE_X: return 1;
+    case eGL_TEXTURE_CUBE_MAP_POSITIVE_Y: return 2;
+    case eGL_TEXTURE_CUBE_MAP_NEGATIVE_Y: return 3;
+    case eGL_TEXTURE_CUBE_MAP_POSITIVE_Z: return 4;
+    case eGL_TEXTURE_CUBE_MAP_NEGATIVE_Z: return 5;
+    default: break;
+  }
+
+  return 0;
+}
+
 GLenum TextureTarget(GLenum target)
 {
   switch(target)

renderdoc/driver/gl/gl_resources.h

@@ -35,6 +35,8 @@ size_t GetByteSize(GLsizei w, GLsizei h, GLsizei d, GLenum format, GLenum type);
 GLenum GetBaseFormat(GLenum internalFormat);
 GLenum GetDataType(GLenum internalFormat);
 GLenum GetSizedFormat(const GLHookSet &gl, GLenum target, GLenum internalFormat);
+void GetFramebufferMipAndLayer(const GLHookSet &gl, GLenum framebuffer, GLenum attachment,
+                               GLint *mip, GLint *layer);
 
 bool EmulateLuminanceFormat(const GLHookSet &gl, GLuint tex, GLenum target, GLenum &internalFormat,
                             GLenum &dataFormat);
@@ -53,6 +55,7 @@ bool IsUIntFormat(GLenum internalFormat);
 bool IsSIntFormat(GLenum internalFormat);
 bool IsSRGBFormat(GLenum internalFormat);
 
+GLint CubeTargetIndex(GLenum face);
 GLenum TextureBinding(GLenum target);
 GLenum TextureTarget(GLenum target);
 bool IsProxyTarget(GLenum target);