Reset stream-out buffer size after failed allocation

renderdoc/driver/d3d11/d3d11_postvs.cpp

@@ -83,6 +83,13 @@ void D3D11Replay::CreateSOBuffers()
   hr = m_pDevice->CreateBuffer(&bufferDesc, NULL, &m_SOStagingBuffer);
   if(FAILED(hr))
     RDCERR("Failed to create m_SOStagingBuffer HRESULT: %s", ToStr(hr).c_str());
+
+  if(!m_SOBuffer || !m_SOStagingBuffer)
+  {
+    SAFE_RELEASE(m_SOBuffer);
+    SAFE_RELEASE(m_SOStagingBuffer);
+    m_SOBufferSize = 0;
+  }
 }
 
 void D3D11Replay::ClearPostVSCache()

renderdoc/driver/d3d12/d3d12_postvs.cpp

@@ -47,6 +47,7 @@ bool D3D12Replay::CreateSOBuffers()
         "Stream-out buffer size %llu is close to or over 4GB, out of memory very likely so "
         "skipping",
         m_SOBufferSize);
+    m_SOBufferSize = 0;
     return false;
   }
 
@@ -79,6 +80,7 @@ bool D3D12Replay::CreateSOBuffers()
   if(FAILED(hr))
   {
     RDCERR("Failed to create SO output buffer, HRESULT: %s", ToStr(hr).c_str());
+    m_SOBufferSize = 0;
     return false;
   }
 
@@ -94,6 +96,7 @@ bool D3D12Replay::CreateSOBuffers()
   if(FAILED(hr))
   {
     RDCERR("Failed to create readback buffer, HRESULT: %s", ToStr(hr).c_str());
+    m_SOBufferSize = 0;
     return false;
   }
 
@@ -111,6 +114,7 @@ bool D3D12Replay::CreateSOBuffers()
   if(FAILED(hr))
   {
     RDCERR("Failed to create SO index buffer, HRESULT: %s", ToStr(hr).c_str());
+    m_SOBufferSize = 0;
     return false;
   }
 
@@ -125,6 +129,7 @@ bool D3D12Replay::CreateSOBuffers()
   if(FAILED(hr))
   {
     RDCERR("Failed to create SO query heap, HRESULT: %s", ToStr(hr).c_str());
+    m_SOBufferSize = 0;
     return false;
   }
 

renderdoc/replay/replay_driver.cpp

@@ -550,6 +550,9 @@ void StandardFillCBufferVariables(ResourceId shader, const rdcarray<ShaderConsta
 
 uint64_t CalcMeshOutputSize(uint64_t curSize, uint64_t requiredOutput)
 {
+  if(curSize == 0)
+    curSize = 32 * 1024 * 1024;
+
   // resize exponentially up to 256MB to avoid repeated resizes
   while(curSize < requiredOutput && curSize < 0x10000000ULL)
     curSize *= 2;