From a091dea6f8647c1d18b7e923ac1b88ca8968584f Mon Sep 17 00:00:00 2001
From: Neil Forbes-Richardson <neilo@neilo.gd>
Date: Thu, 14 Dec 2017 17:09:18 +1100
Subject: [PATCH] D3D11 texture serialisation size estimation fix.

During size estimation phase of serialisation, the row + depth pitch were not conservative enough estimates on some drivers. To remedy this, it's determined exactly at estimation time by attempting to map the resources.
---
 renderdoc/driver/d3d11/d3d11_initstate.cpp | 24 ++++++++++++++++++----
 renderdoc/driver/d3d11/d3d11_resources.cpp | 23 +++++++++++++++++++++
 renderdoc/driver/d3d11/d3d11_resources.h   |  9 ++++++++
 3 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/renderdoc/driver/d3d11/d3d11_initstate.cpp b/renderdoc/driver/d3d11/d3d11_initstate.cpp
index 1f6483d27..f19cb0004 100644
--- a/renderdoc/driver/d3d11/d3d11_initstate.cpp
+++ b/renderdoc/driver/d3d11/d3d11_initstate.cpp
@@ -280,12 +280,14 @@ uint32_t WrappedID3D11Device::GetSize_InitialState(ResourceId id, ID3D11DeviceCh
 
   uint32_t ret = 128;    // type, Id, plus breathing room
 
-  // pessimistic RowPitch alignment since we don't know what will be reported
+  // pessimistic RowPitch alignment where we can't determine it correctly.
   const UINT WorstRowPitchAlign = 256;
 
   // pessimistic DepthPitch alignment
   const UINT WorstDepthPitchAlign = 256;
 
+  ResourcePitch pitch = {WorstRowPitchAlign, WorstDepthPitchAlign};
+
   D3D11ResourceType type = IdentifyTypeByPtr(res);
 
   if(type == Resource_UnorderedAccessView)
@@ -348,6 +350,8 @@ uint32_t WrappedID3D11Device::GetSize_InitialState(ResourceId id, ID3D11DeviceCh
       ret += 4;                      // number of subresources
       ret += 4 * NumSubresources;    // RowPitch for each subresource
 
+      ID3D11Resource *stage = (ID3D11Resource *)m_ResourceManager->GetInitialContents(id).resource;
+
       // Subresource contents:
       for(UINT sub = 0; sub < NumSubresources; sub++)
       {
@@ -357,8 +361,13 @@ uint32_t WrappedID3D11Device::GetSize_InitialState(ResourceId id, ID3D11DeviceCh
         if(IsBlockFormat(desc.Format))
           numRows = AlignUp4(numRows) / 4;
 
+        if(stage)
+          pitch = GetResourcePitchForSubresource(m_pImmediateContext->GetReal(), stage, sub);
+        else
+          pitch = {WorstRowPitchAlign, WorstDepthPitchAlign};
+
         const UINT RowPitch = GetByteSize(desc.Width, 1, 1, desc.Format, mip);
-        const UINT WorstRowPitch = AlignUp(RowPitch, WorstRowPitchAlign);
+        const UINT WorstRowPitch = AlignUp(RowPitch, pitch.m_RowPitch);
 
         ret += WorstRowPitch * numRows;
         ret += (uint32_t)WriteSerialiser::GetChunkAlignment();
@@ -377,20 +386,27 @@ uint32_t WrappedID3D11Device::GetSize_InitialState(ResourceId id, ID3D11DeviceCh
     ret += 4;                      // number of subresources
     ret += 8 * NumSubresources;    // RowPitch and DepthPitch for each subresource
 
+    ID3D11Resource *stage = (ID3D11Resource *)m_ResourceManager->GetInitialContents(id).resource;
+
     // Subresource contents:
     for(UINT sub = 0; sub < NumSubresources; sub++)
     {
       UINT mip = GetMipForSubresource(tex, sub);
 
+      if(stage)
+        pitch = GetResourcePitchForSubresource(m_pImmediateContext->GetReal(), stage, sub);
+      else
+        pitch = {WorstRowPitchAlign, WorstDepthPitchAlign};
+
       uint32_t numRows = RDCMAX(1U, desc.Height >> mip);
       if(IsBlockFormat(desc.Format))
         numRows = AlignUp4(numRows) / 4;
 
       const UINT RowPitch = GetByteSize(desc.Width, 1, 1, desc.Format, mip);
-      const UINT WorstRowPitch = AlignUp(RowPitch, WorstRowPitchAlign);
+      const UINT WorstRowPitch = AlignUp(RowPitch, pitch.m_RowPitch);
 
       const UINT DepthPitch = WorstRowPitch * numRows;
-      const UINT WorstDepthPitch = AlignUp(DepthPitch, WorstDepthPitchAlign);
+      const UINT WorstDepthPitch = AlignUp(DepthPitch, pitch.m_DepthPitch);
 
       ret += WorstDepthPitch * RDCMAX(1U, desc.Depth >> mip);
       ret += (uint32_t)WriteSerialiser::GetChunkAlignment();
diff --git a/renderdoc/driver/d3d11/d3d11_resources.cpp b/renderdoc/driver/d3d11/d3d11_resources.cpp
index 393b61162..2e4be57ea 100644
--- a/renderdoc/driver/d3d11/d3d11_resources.cpp
+++ b/renderdoc/driver/d3d11/d3d11_resources.cpp
@@ -250,6 +250,29 @@ UINT GetMipForSubresource(ID3D11Resource *res, int Subresource)
   return mipLevel;
 }
 
+ResourcePitch GetResourcePitchForSubresource(ID3D11DeviceContext *ctx, ID3D11Resource *res,
+                                             int Subresource)
+{
+  ResourcePitch pitch = {};
+  D3D11_MAPPED_SUBRESOURCE mapped = {};
+  HRESULT hr = E_INVALIDARG;
+
+  hr = ctx->Map(res, Subresource, D3D11_MAP_READ, 0, &mapped);
+
+  if(FAILED(hr))
+  {
+    RDCERR("Failed to map while getting resource pitch HRESULT: %s", ToStr(hr).c_str());
+  }
+  else
+  {
+    pitch.m_RowPitch = mapped.RowPitch;
+    pitch.m_DepthPitch = mapped.DepthPitch;
+    ctx->Unmap(res, Subresource);
+  }
+
+  return pitch;
+}
+
 UINT GetByteSize(ID3D11Texture1D *tex, int SubResource)
 {
   D3D11_TEXTURE1D_DESC desc;
diff --git a/renderdoc/driver/d3d11/d3d11_resources.h b/renderdoc/driver/d3d11/d3d11_resources.h
index 0860fa0e5..d0c468ab4 100644
--- a/renderdoc/driver/d3d11/d3d11_resources.h
+++ b/renderdoc/driver/d3d11/d3d11_resources.h
@@ -75,6 +75,15 @@ UINT GetByteSize(ID3D11Texture3D *tex, int SubResource);
 
 UINT GetMipForSubresource(ID3D11Resource *res, int Subresource);
 
+struct ResourcePitch
+{
+  UINT m_RowPitch;
+  UINT m_DepthPitch;
+};
+
+ResourcePitch GetResourcePitchForSubresource(ID3D11DeviceContext *ctx, ID3D11Resource *res,
+                                             int Subresource);
+
 template <typename derived, typename base>
 bool CanQuery(base *b)
 {