From 81ee4e18a8a486ee2f2873a3e94e9f24bd68eb55 Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Thu, 9 Apr 2020 21:08:34 +0100
Subject: [PATCH] Reset stream-out buffer size after failed allocation

---
 renderdoc/driver/d3d11/d3d11_postvs.cpp | 7 +++++++
 renderdoc/driver/d3d12/d3d12_postvs.cpp | 5 +++++
 renderdoc/replay/replay_driver.cpp      | 3 +++
 3 files changed, 15 insertions(+)

diff --git a/renderdoc/driver/d3d11/d3d11_postvs.cpp b/renderdoc/driver/d3d11/d3d11_postvs.cpp
index 59fd43b2b..f946211e3 100644
--- a/renderdoc/driver/d3d11/d3d11_postvs.cpp
+++ b/renderdoc/driver/d3d11/d3d11_postvs.cpp
@@ -83,6 +83,13 @@ void D3D11Replay::CreateSOBuffers()
   hr = m_pDevice->CreateBuffer(&bufferDesc, NULL, &m_SOStagingBuffer);
   if(FAILED(hr))
     RDCERR("Failed to create m_SOStagingBuffer HRESULT: %s", ToStr(hr).c_str());
+
+  if(!m_SOBuffer || !m_SOStagingBuffer)
+  {
+    SAFE_RELEASE(m_SOBuffer);
+    SAFE_RELEASE(m_SOStagingBuffer);
+    m_SOBufferSize = 0;
+  }
 }
 
 void D3D11Replay::ClearPostVSCache()
diff --git a/renderdoc/driver/d3d12/d3d12_postvs.cpp b/renderdoc/driver/d3d12/d3d12_postvs.cpp
index e637339a2..c7300ff80 100644
--- a/renderdoc/driver/d3d12/d3d12_postvs.cpp
+++ b/renderdoc/driver/d3d12/d3d12_postvs.cpp
@@ -47,6 +47,7 @@ bool D3D12Replay::CreateSOBuffers()
         "Stream-out buffer size %llu is close to or over 4GB, out of memory very likely so "
         "skipping",
         m_SOBufferSize);
+    m_SOBufferSize = 0;
     return false;
   }
 
@@ -79,6 +80,7 @@ bool D3D12Replay::CreateSOBuffers()
   if(FAILED(hr))
   {
     RDCERR("Failed to create SO output buffer, HRESULT: %s", ToStr(hr).c_str());
+    m_SOBufferSize = 0;
     return false;
   }
 
@@ -94,6 +96,7 @@ bool D3D12Replay::CreateSOBuffers()
   if(FAILED(hr))
   {
     RDCERR("Failed to create readback buffer, HRESULT: %s", ToStr(hr).c_str());
+    m_SOBufferSize = 0;
     return false;
   }
 
@@ -111,6 +114,7 @@ bool D3D12Replay::CreateSOBuffers()
   if(FAILED(hr))
   {
     RDCERR("Failed to create SO index buffer, HRESULT: %s", ToStr(hr).c_str());
+    m_SOBufferSize = 0;
     return false;
   }
 
@@ -125,6 +129,7 @@ bool D3D12Replay::CreateSOBuffers()
   if(FAILED(hr))
   {
     RDCERR("Failed to create SO query heap, HRESULT: %s", ToStr(hr).c_str());
+    m_SOBufferSize = 0;
     return false;
   }
 
diff --git a/renderdoc/replay/replay_driver.cpp b/renderdoc/replay/replay_driver.cpp
index 2bf1fa481..bbf401ff0 100644
--- a/renderdoc/replay/replay_driver.cpp
+++ b/renderdoc/replay/replay_driver.cpp
@@ -550,6 +550,9 @@ void StandardFillCBufferVariables(ResourceId shader, const rdcarray<ShaderConsta
 
 uint64_t CalcMeshOutputSize(uint64_t curSize, uint64_t requiredOutput)
 {
+  if(curSize == 0)
+    curSize = 32 * 1024 * 1024;
+
   // resize exponentially up to 256MB to avoid repeated resizes
   while(curSize < requiredOutput && curSize < 0x10000000ULL)
     curSize *= 2;