Only flip uncompressed texture data out of GL when saving to disk

* When transferring over the network, we keep texture data consistently
  in GL origin-bottom-left order. This means we can just flip images on
  display and otherwise have things consistently behaving, while still
  preserving the behaviour of flipping on saving to disk to try and
  mostly 'do the right thing' when saving an image.
* The behaviour should be the same as before except for remote proxying
  which is fixed. The behaviour for GL is still that compressed images
  saved as compressed will appear to flip vertically from what is
  natively displayed in the UI, but I think this is the only sensible
  way to behave (and anyway, flipping compressed images is far too
  involved to be worthwhile).

qrenderdoc/Windows/TextureViewer.cpp

@@ -235,11 +235,11 @@ void TextureViewer::OnEventSelected(uint32_t eventID)
 
     TextureDisplay &d = m_TexDisplay;
 
-    if(m_Core->APIProps().pipelineType == ePipelineState_D3D11)
+    if(m_Core->APIProps().pipelineType == eGraphicsAPI_D3D11)
     {
       d.texid = m_Core->CurD3D11PipelineState.m_OM.RenderTargets[0].Resource;
     }
-    else if(m_Core->APIProps().pipelineType == ePipelineState_OpenGL)
+    else if(m_Core->APIProps().pipelineType == eGraphicsAPI_OpenGL)
     {
       d.texid = m_Core->CurGLPipelineState.m_FB.m_DrawFBO.Color[0].Obj;
     }

renderdoc/core/image_viewer.cpp

@@ -140,12 +140,12 @@ public:
   }
   vector<ResourceId> GetTextures() { return m_Proxy->GetTextures(); }
   FetchTexture GetTexture(ResourceId id) { return m_Proxy->GetTexture(m_TextureID); }
-  byte *GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
+  byte *GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip, bool forDiskSave,
                        FormatComponentType typeHint, bool resolve, bool forceRGBA8unorm,
                        float blackPoint, float whitePoint, size_t &dataSize)
   {
-    return m_Proxy->GetTextureData(m_TextureID, arrayIdx, mip, typeHint, resolve, forceRGBA8unorm,
-                                   blackPoint, whitePoint, dataSize);
+    return m_Proxy->GetTextureData(m_TextureID, arrayIdx, mip, forDiskSave, typeHint, resolve,
+                                   forceRGBA8unorm, blackPoint, whitePoint, dataSize);
   }
 
   // handle a couple of operations ourselves to return a simple fake log

renderdoc/core/replay_proxy.cpp

@@ -1719,7 +1719,8 @@ void ReplayProxy::EnsureTexCached(ResourceId texid, uint32_t arrayIdx, uint32_t
     ResourceId proxyid = m_ProxyTextureIds[texid];
 
     size_t size;
-    byte *data = GetTextureData(texid, arrayIdx, mip, eCompType_None, false, false, 0.0f, 0.0f, size);
+    byte *data =
+        GetTextureData(texid, arrayIdx, mip, false, eCompType_None, false, false, 0.0f, 0.0f, size);
 
     if(data)
       m_Proxy->SetProxyTextureData(proxyid, arrayIdx, mip, data, size);
@@ -1818,7 +1819,7 @@ bool ReplayProxy::Tick(int type, Serialiser *incomingPacket)
     case eReplayProxy_GetTextureData:
     {
       size_t dummy;
-      GetTextureData(ResourceId(), 0, 0, eCompType_None, false, false, 0.0f, 0.0f, dummy);
+      GetTextureData(ResourceId(), 0, 0, false, eCompType_None, false, false, 0.0f, 0.0f, dummy);
       break;
     }
     case eReplayProxy_InitPostVS: InitPostVSBuffers(0); break;
@@ -1892,13 +1893,15 @@ APIProperties ReplayProxy::GetAPIProperties()
   {
     if(!SendReplayCommand(eReplayProxy_GetAPIProperties))
       return ret;
-
-    APIProperties local = m_Proxy->GetAPIProperties();
-    ret.localRenderer = local.localRenderer;
   }
 
   m_FromReplaySerialiser->Serialise("", ret);
 
+  if(!m_RemoteServer)
+    ret.localRenderer = m_Proxy->GetAPIProperties().localRenderer;
+
+  m_APIProps = ret;
+
   return ret;
 }
 
@@ -2291,13 +2294,14 @@ void ReplayProxy::GetBufferData(ResourceId buff, uint64_t offset, uint64_t len,
   }
 }
 
-byte *ReplayProxy::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
+byte *ReplayProxy::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip, bool forDiskSave,
                                   FormatComponentType typeHint, bool resolve, bool forceRGBA8unorm,
                                   float blackPoint, float whitePoint, size_t &dataSize)
 {
   m_ToReplaySerialiser->Serialise("", tex);
   m_ToReplaySerialiser->Serialise("", arrayIdx);
   m_ToReplaySerialiser->Serialise("", mip);
+  m_ToReplaySerialiser->Serialise("", forDiskSave);
   m_ToReplaySerialiser->Serialise("", typeHint);
   m_ToReplaySerialiser->Serialise("", resolve);
   m_ToReplaySerialiser->Serialise("", forceRGBA8unorm);
@@ -2306,8 +2310,8 @@ byte *ReplayProxy::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mi
 
   if(m_RemoteServer)
   {
-    byte *data = m_Remote->GetTextureData(tex, arrayIdx, mip, typeHint, resolve, forceRGBA8unorm,
-                                          blackPoint, whitePoint, dataSize);
+    byte *data = m_Remote->GetTextureData(tex, arrayIdx, mip, forDiskSave, typeHint, resolve,
+                                          forceRGBA8unorm, blackPoint, whitePoint, dataSize);
 
     byte *compressed = new byte[LZ4_COMPRESSBOUND(dataSize)];
 

renderdoc/core/replay_proxy.h

@@ -103,6 +103,8 @@ public:
     m_FromReplaySerialiser = NULL;
     m_ToReplaySerialiser = new Serialiser(NULL, Serialiser::WRITING, false);
     m_RemoteHasResolver = false;
+
+    GetAPIProperties();
   }
 
   ReplayProxy(Network::Socket *sock, IRemoteDriver *remote)
@@ -224,6 +226,15 @@ public:
       if(cfg.texid == ResourceId() || m_ProxyTextureIds[cfg.texid] == ResourceId())
         return false;
       cfg.texid = m_ProxyTextureIds[cfg.texid];
+
+      // due to OpenGL having origin bottom-left compared to the rest of the world,
+      // we need to flip going in or out of GL.
+      if((m_APIProps.pipelineType == eGraphicsAPI_OpenGL) !=
+         (m_APIProps.localRenderer == eGraphicsAPI_OpenGL))
+      {
+        cfg.FlipY = !cfg.FlipY;
+      }
+
       return m_Proxy->RenderTexture(cfg);
     }
 
@@ -238,7 +249,22 @@ public:
       EnsureTexCached(texture, sliceFace, mip);
       if(texture == ResourceId() || m_ProxyTextureIds[texture] == ResourceId())
         return;
-      m_Proxy->PickPixel(m_ProxyTextureIds[texture], x, y, sliceFace, mip, sample, typeHint, pixel);
+
+      texture = m_ProxyTextureIds[texture];
+
+      // due to OpenGL having origin bottom-left compared to the rest of the world,
+      // we need to flip going in or out of GL.
+      // This is a bit more annoying here as we don't have a bool to flip, we need to
+      // manually adjust y
+      if((m_APIProps.pipelineType == eGraphicsAPI_OpenGL) !=
+         (m_APIProps.localRenderer == eGraphicsAPI_OpenGL))
+      {
+        FetchTexture tex = m_Proxy->GetTexture(texture);
+        uint32_t mipHeight = RDCMAX(1U, tex.height >> mip);
+        y = (mipHeight - 1) - y;
+      }
+
+      m_Proxy->PickPixel(texture, x, y, sliceFace, mip, sample, typeHint, pixel);
     }
   }
 
@@ -393,7 +419,7 @@ public:
                             vector<ShaderVariable> &outvars, const vector<byte> &data);
 
   void GetBufferData(ResourceId buff, uint64_t offset, uint64_t len, vector<byte> &retData);
-  byte *GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
+  byte *GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip, bool forDiskSave,
                        FormatComponentType typeHint, bool resolve, bool forceRGBA8unorm,
                        float blackPoint, float whitePoint, size_t &dataSize);
 
@@ -510,6 +536,8 @@ private:
 
   bool m_RemoteHasResolver;
 
+  APIProperties m_APIProps;
+
   D3D11PipelineState m_D3D11PipelineState;
   GLPipelineState m_GLPipelineState;
   VulkanPipelineState m_VulkanPipelineState;

renderdoc/driver/d3d11/d3d11_analyse.cpp

@@ -2519,9 +2519,9 @@ void D3D11DebugManager::PickPixel(ResourceId texture, uint32_t x, uint32_t y, ui
 }
 
 byte *D3D11DebugManager::GetTextureData(ResourceId id, uint32_t arrayIdx, uint32_t mip,
-                                        FormatComponentType typeHint, bool resolve,
-                                        bool forceRGBA8unorm, float blackPoint, float whitePoint,
-                                        size_t &dataSize)
+                                        bool forDiskSave, FormatComponentType typeHint,
+                                        bool resolve, bool forceRGBA8unorm, float blackPoint,
+                                        float whitePoint, size_t &dataSize)
 {
   ID3D11Resource *dummyTex = NULL;
 

renderdoc/driver/d3d11/d3d11_debug.h

@@ -127,7 +127,7 @@ public:
   void GetBufferData(ID3D11Buffer *buff, uint64_t offset, uint64_t length, vector<byte> &retData,
                      bool unwrap);
 
-  byte *GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
+  byte *GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip, bool forDiskSave,
                        FormatComponentType typeHint, bool resolve, bool forceRGBA8unorm,
                        float blackPoint, float whitePoint, size_t &dataSize);
 

renderdoc/driver/d3d11/d3d11_replay.cpp

@@ -1450,12 +1450,13 @@ void D3D11Replay::GetBufferData(ResourceId buff, uint64_t offset, uint64_t len,
   m_pDevice->GetDebugManager()->GetBufferData(buff, offset, len, retData);
 }
 
-byte *D3D11Replay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
+byte *D3D11Replay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip, bool forDiskSave,
                                   FormatComponentType typeHint, bool resolve, bool forceRGBA8unorm,
                                   float blackPoint, float whitePoint, size_t &dataSize)
 {
-  return m_pDevice->GetDebugManager()->GetTextureData(
-      tex, arrayIdx, mip, typeHint, resolve, forceRGBA8unorm, blackPoint, whitePoint, dataSize);
+  return m_pDevice->GetDebugManager()->GetTextureData(tex, arrayIdx, mip, forDiskSave, typeHint,
+                                                      resolve, forceRGBA8unorm, blackPoint,
+                                                      whitePoint, dataSize);
 }
 
 void D3D11Replay::ReplaceResource(ResourceId from, ResourceId to)

renderdoc/driver/d3d11/d3d11_replay.h

@@ -106,7 +106,7 @@ public:
   MeshFormat GetPostVSBuffers(uint32_t eventID, uint32_t instID, MeshDataStage stage);
 
   void GetBufferData(ResourceId buff, uint64_t offset, uint64_t len, vector<byte> &retData);
-  byte *GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
+  byte *GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip, bool forDiskSave,
                        FormatComponentType typeHint, bool resolve, bool forceRGBA8unorm,
                        float blackPoint, float whitePoint, size_t &dataSize);
 

renderdoc/driver/d3d12/d3d12_replay.cpp

@@ -388,7 +388,7 @@ void D3D12Replay::GetBufferData(ResourceId buff, uint64_t offset, uint64_t len,
 {
 }
 
-byte *D3D12Replay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
+byte *D3D12Replay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip, bool forDiskSave,
                                   FormatComponentType typeHint, bool resolve, bool forceRGBA8unorm,
                                   float blackPoint, float whitePoint, size_t &dataSize)
 {

renderdoc/driver/d3d12/d3d12_replay.h

@@ -101,7 +101,7 @@ public:
   MeshFormat GetPostVSBuffers(uint32_t eventID, uint32_t instID, MeshDataStage stage);
 
   void GetBufferData(ResourceId buff, uint64_t offset, uint64_t len, vector<byte> &retData);
-  byte *GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
+  byte *GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip, bool forDiskSave,
                        FormatComponentType typeHint, bool resolve, bool forceRGBA8unorm,
                        float blackPoint, float whitePoint, size_t &dataSize);
 

renderdoc/driver/gl/gl_replay.cpp

@@ -2216,7 +2216,7 @@ void GLReplay::FillCBufferVariables(ResourceId shader, string entryPoint, uint32
                        outvars, data);
 }
 
-byte *GLReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
+byte *GLReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip, bool forDiskSave,
                                FormatComponentType typeHint, bool resolve, bool forceRGBA8unorm,
                                float blackPoint, float whitePoint, size_t &dataSize)
 {
@@ -2487,62 +2487,72 @@ byte *GLReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
 
       m_pDriver->glGetTexImage(target, (GLint)mip, fmt, type, ret);
 
-      // need to vertically flip the image now to get conventional row ordering
-      // we either do this when copying out the slice of interest, or just
-      // on its own
-      size_t rowSize = GetByteSize(width, 1, 1, fmt, type);
-      byte *src, *dst;
+      // if we're saving to disk we make the decision to vertically flip any non-compressed
+      // images. This is a bit arbitrary, but really origin top-left is common for all disk
+      // formats so we do this flip from bottom-left origin. We only do this for saving to
+      // disk so that if we're transferring over the network etc for remote replay, the image
+      // order is consistent (and we just need to take care to apply an extra vertical flip
+      // for display when proxying).
 
-      // for arrays just extract the slice we're interested in.
-      if(texType == eGL_TEXTURE_2D_ARRAY || texType == eGL_TEXTURE_1D_ARRAY ||
-         texType == eGL_TEXTURE_CUBE_MAP_ARRAY)
+      if(forDiskSave)
       {
-        dataSize = GetByteSize(width, height, 1, fmt, type);
-        byte *slice = new byte[dataSize];
+        // need to vertically flip the image now to get conventional row ordering
+        // we either do this when copying out the slice of interest, or just
+        // on its own
+        size_t rowSize = GetByteSize(width, 1, 1, fmt, type);
+        byte *src, *dst;
 
-        // src points to the last row in the array slice image
-        src = (ret + dataSize * arrayIdx) + (height - 1) * rowSize;
-        dst = slice;
-
-        // we do memcpy + vertical flip
-        // memcpy(slice, ret + dataSize*arrayIdx, dataSize);
-
-        for(GLsizei i = 0; i < height; i++)
+        // for arrays just extract the slice we're interested in.
+        if(texType == eGL_TEXTURE_2D_ARRAY || texType == eGL_TEXTURE_1D_ARRAY ||
+           texType == eGL_TEXTURE_CUBE_MAP_ARRAY)
         {
-          memcpy(dst, src, rowSize);
+          dataSize = GetByteSize(width, height, 1, fmt, type);
+          byte *slice = new byte[dataSize];
 
-          dst += rowSize;
-          src -= rowSize;
-        }
+          // src points to the last row in the array slice image
+          src = (ret + dataSize * arrayIdx) + (height - 1) * rowSize;
+          dst = slice;
 
-        delete[] ret;
+          // we do memcpy + vertical flip
+          // memcpy(slice, ret + dataSize*arrayIdx, dataSize);
 
-        ret = slice;
-      }
-      else
-      {
-        byte *row = new byte[rowSize];
-
-        size_t sliceSize = GetByteSize(width, height, 1, fmt, type);
-
-        // invert all slices in a 3D texture
-        for(GLsizei d = 0; d < depth; d++)
-        {
-          dst = ret + d * sliceSize;
-          src = dst + (height - 1) * rowSize;
-
-          for(GLsizei i = 0; i<height>> 1; i++)
+          for(GLsizei i = 0; i < height; i++)
           {
-            memcpy(row, src, rowSize);
-            memcpy(src, dst, rowSize);
-            memcpy(dst, row, rowSize);
+            memcpy(dst, src, rowSize);
 
             dst += rowSize;
             src -= rowSize;
           }
-        }
 
-        delete[] row;
+          delete[] ret;
+
+          ret = slice;
+        }
+        else
+        {
+          byte *row = new byte[rowSize];
+
+          size_t sliceSize = GetByteSize(width, height, 1, fmt, type);
+
+          // invert all slices in a 3D texture
+          for(GLsizei d = 0; d < depth; d++)
+          {
+            dst = ret + d * sliceSize;
+            src = dst + (height - 1) * rowSize;
+
+            for(GLsizei i = 0; i<height>> 1; i++)
+            {
+              memcpy(row, src, rowSize);
+              memcpy(src, dst, rowSize);
+              memcpy(dst, row, rowSize);
+
+              dst += rowSize;
+              src -= rowSize;
+            }
+          }
+
+          delete[] row;
+        }
       }
     }
 

renderdoc/driver/gl/gl_replay.h

@@ -151,7 +151,7 @@ public:
   MeshFormat GetPostVSBuffers(uint32_t eventID, uint32_t instID, MeshDataStage stage);
 
   void GetBufferData(ResourceId buff, uint64_t offset, uint64_t len, vector<byte> &ret);
-  byte *GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
+  byte *GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip, bool forDiskSave,
                        FormatComponentType typeHint, bool resolve, bool forceRGBA8unorm,
                        float blackPoint, float whitePoint, size_t &dataSize);
 

renderdoc/driver/vulkan/vk_replay.cpp

@@ -4568,7 +4568,7 @@ MeshFormat VulkanReplay::GetPostVSBuffers(uint32_t eventID, uint32_t instID, Mes
   return GetDebugManager()->GetPostVSBuffers(eventID, instID, stage);
 }
 
-byte *VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
+byte *VulkanReplay::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip, bool forDiskSave,
                                    FormatComponentType typeHint, bool resolve, bool forceRGBA8unorm,
                                    float blackPoint, float whitePoint, size_t &dataSize)
 {

renderdoc/driver/vulkan/vk_replay.h

@@ -186,7 +186,7 @@ public:
   MeshFormat GetPostVSBuffers(uint32_t eventID, uint32_t instID, MeshDataStage stage);
 
   void GetBufferData(ResourceId buff, uint64_t offset, uint64_t len, vector<byte> &retData);
-  byte *GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
+  byte *GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip, bool forDiskSave,
                        FormatComponentType typeHint, bool resolve, bool forceRGBA8unorm,
                        float blackPoint, float whitePoint, size_t &dataSize);
 

renderdoc/replay/replay_driver.h

@@ -88,7 +88,7 @@ public:
 
   virtual void GetBufferData(ResourceId buff, uint64_t offset, uint64_t len,
                              vector<byte> &retData) = 0;
-  virtual byte *GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
+  virtual byte *GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip, bool forDiskSave,
                                FormatComponentType typeHint, bool resolve, bool forceRGBA8unorm,
                                float blackPoint, float whitePoint, size_t &dataSize) = 0;
 

renderdoc/replay/replay_renderer.cpp

@@ -483,8 +483,8 @@ bool ReplayRenderer::GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t
   }
 
   size_t sz = 0;
-  byte *bytes =
-      m_pDevice->GetTextureData(liveId, arrayIdx, mip, eCompType_None, false, false, 0.0f, 0.0f, sz);
+  byte *bytes = m_pDevice->GetTextureData(liveId, arrayIdx, mip, false, eCompType_None, false,
+                                          false, 0.0f, 0.0f, sz);
 
   if(sz == 0 || bytes == NULL)
     create_array_uninit(*data, 0);
@@ -745,7 +745,7 @@ bool ReplayRenderer::SaveTexture(const TextureSave &saveData, const char *path)
 
       size_t datasize = 0;
       byte *bytes =
-          m_pDevice->GetTextureData(liveid, slice, mip, sd.typeHint, resolveSamples, downcast,
+          m_pDevice->GetTextureData(liveid, slice, mip, true, sd.typeHint, resolveSamples, downcast,
                                     sd.comp.blackPoint, sd.comp.whitePoint, datasize);
 
       if(bytes == NULL)