Don't treat OOM in post-vs code as fatal, since we handle it gracefully

renderdoc/driver/d3d11/d3d11_device.cpp

@@ -2535,8 +2535,15 @@ void WrappedID3D11Device::CheckHRESULT(HRESULT hr)
   }
   else if(hr == E_OUTOFMEMORY)
   {
-    RDCLOG("Logging out of memory fatal error for %s", ToStr(hr).c_str());
-    m_FatalError = ReplayStatus::ReplayOutOfMemory;
+    if(m_OOMHandler)
+    {
+      RDCLOG("Ignoring out of memory error that will be handled");
+    }
+    else
+    {
+      RDCLOG("Logging out of memory fatal error for %s", ToStr(hr).c_str());
+      m_FatalError = ReplayStatus::ReplayOutOfMemory;
+    }
   }
   else
   {

renderdoc/driver/d3d11/d3d11_device.h

@@ -600,6 +600,7 @@ private:
   SDFile *m_StructuredFile = NULL;
   SDFile *m_StoredStructuredData;
 
+  int m_OOMHandler = 0;
   rdcarray<DebugMessage> m_DebugMessages;
   ReplayStatus m_FatalError = ReplayStatus::Succeeded;
 
@@ -673,6 +674,13 @@ public:
   double GetTimeFrequency() { return m_TimeFrequency; }
   void FirstFrame(IDXGISwapper *swapper);
 
+  void HandleOOM(bool handle)
+  {
+    if(handle)
+      m_OOMHandler++;
+    else
+      m_OOMHandler--;
+  }
   void CheckHRESULT(HRESULT hr);
   void ReportFatalError(ReplayStatus error) { m_FatalError = error; }
   ReplayStatus FatalErrorCheck() { return m_FatalError; }

renderdoc/driver/d3d11/d3d11_postvs.cpp

@@ -32,6 +32,18 @@
 #include "d3d11_replay.h"
 #include "d3d11_resources.h"
 
+struct ScopedOOMHandle11
+{
+  ScopedOOMHandle11(WrappedID3D11Device *dev)
+  {
+    m_pDevice = dev;
+    m_pDevice->HandleOOM(true);
+  }
+
+  ~ScopedOOMHandle11() { m_pDevice->HandleOOM(false); }
+  WrappedID3D11Device *m_pDevice;
+};
+
 void D3D11Replay::InitStreamOut()
 {
   CreateSOBuffers();
@@ -179,6 +191,9 @@ void D3D11Replay::InitPostVSBuffers(uint32_t eventId)
   if(m_PostVSData.find(eventId) != m_PostVSData.end())
     return;
 
+  // we handle out-of-memory errors while processing postvs, don't treat it as a fatal error
+  ScopedOOMHandle11 oom(m_pDevice);
+
   D3D11MarkerRegion postvs(StringFormat::Fmt("PostVS for %u", eventId));
 
   D3D11RenderStateTracker tracker(m_pImmediateContext);

renderdoc/driver/d3d12/d3d12_device.cpp

@@ -3022,8 +3022,15 @@ void WrappedID3D12Device::CheckHRESULT(HRESULT hr)
   }
   else if(hr == E_OUTOFMEMORY)
   {
-    RDCLOG("Logging out of memory fatal error for %s", ToStr(hr).c_str());
-    m_FatalError = ReplayStatus::ReplayOutOfMemory;
+    if(m_OOMHandler)
+    {
+      RDCLOG("Ignoring out of memory error that will be handled");
+    }
+    else
+    {
+      RDCLOG("Logging out of memory fatal error for %s", ToStr(hr).c_str());
+      m_FatalError = ReplayStatus::ReplayOutOfMemory;
+    }
   }
   else
   {

renderdoc/driver/d3d12/d3d12_device.h

@@ -667,6 +667,7 @@ private:
   rdcarray<TempMem *> m_ThreadTempMem;
 
   rdcarray<DebugMessage> m_DebugMessages;
+  int m_OOMHandler = 0;
   ReplayStatus m_FatalError = ReplayStatus::Succeeded;
 
   uint64_t m_TimeBase = 0;
@@ -801,6 +802,13 @@ public:
   void AddDebugMessage(const DebugMessage &msg);
   rdcarray<DebugMessage> GetDebugMessages();
 
+  void HandleOOM(bool handle)
+  {
+    if(handle)
+      m_OOMHandler++;
+    else
+      m_OOMHandler--;
+  }
   void CheckHRESULT(HRESULT hr);
   void ReportFatalError(ReplayStatus error) { m_FatalError = error; }
   ReplayStatus FatalErrorCheck() { return m_FatalError; }

renderdoc/driver/d3d12/d3d12_postvs.cpp

@@ -32,6 +32,18 @@
 #include "d3d12_replay.h"
 #include "d3d12_shader_cache.h"
 
+struct ScopedOOMHandle12
+{
+  ScopedOOMHandle12(WrappedID3D12Device *dev)
+  {
+    m_pDevice = dev;
+    m_pDevice->HandleOOM(true);
+  }
+
+  ~ScopedOOMHandle12() { m_pDevice->HandleOOM(false); }
+  WrappedID3D12Device *m_pDevice;
+};
+
 bool D3D12Replay::CreateSOBuffers()
 {
   HRESULT hr = S_OK;
@@ -170,6 +182,9 @@ void D3D12Replay::InitPostVSBuffers(uint32_t eventId)
   if(m_PostVSData.find(eventId) != m_PostVSData.end())
     return;
 
+  // we handle out-of-memory errors while processing postvs, don't treat it as a fatal error
+  ScopedOOMHandle12 oom(m_pDevice);
+
   D3D12MarkerRegion postvs(m_pDevice->GetQueue(), StringFormat::Fmt("PostVS for %u", eventId));
 
   D3D12CommandData *cmd = m_pDevice->GetQueue()->GetCommandData();

renderdoc/driver/vulkan/vk_core.cpp

@@ -4009,8 +4009,15 @@ void WrappedVulkan::CheckErrorVkResult(VkResult vkr)
   }
   else if(vkr == VK_ERROR_OUT_OF_HOST_MEMORY || vkr == VK_ERROR_OUT_OF_DEVICE_MEMORY)
   {
-    RDCLOG("Logging out of memory fatal error for %s", ToStr(vkr).c_str());
-    m_FailedReplayStatus = m_FatalError = ReplayStatus::ReplayOutOfMemory;
+    if(m_OOMHandler)
+    {
+      RDCLOG("Ignoring out of memory error that will be handled");
+    }
+    else
+    {
+      RDCLOG("Logging out of memory fatal error for %s", ToStr(vkr).c_str());
+      m_FailedReplayStatus = m_FatalError = ReplayStatus::ReplayOutOfMemory;
+    }
   }
   else
   {

renderdoc/driver/vulkan/vk_core.h

@@ -289,6 +289,7 @@ private:
   rdcarray<DebugMessage> GetDebugMessages();
   void AddDebugMessage(DebugMessage msg);
 
+  int m_OOMHandler = 0;
   ReplayStatus m_FatalError = ReplayStatus::Succeeded;
   CaptureState m_State;
   bool m_AppControlledCapture = false;
@@ -1110,6 +1111,13 @@ public:
   bool SelectGraphicsComputeQueue(const rdcarray<VkQueueFamilyProperties> &queueProps,
                                   VkDeviceCreateInfo &createInfo, uint32_t &queueFamilyIndex);
 
+  void HandleOOM(bool handle)
+  {
+    if(handle)
+      m_OOMHandler++;
+    else
+      m_OOMHandler--;
+  }
   ReplayStatus FatalErrorCheck() { return m_FatalError; }
   bool HasFatalError() { return m_FatalError != ReplayStatus::Succeeded; }
   inline void CheckVkResult(VkResult vkr)

renderdoc/driver/vulkan/vk_postvs.cpp

@@ -39,6 +39,18 @@ RDOC_EXTERN_CONFIG(bool, Vulkan_Debug_DisableBufferDeviceAddress);
 
 #undef None
 
+struct ScopedOOMHandleVk
+{
+  ScopedOOMHandleVk(WrappedVulkan *vk)
+  {
+    m_pDriver = vk;
+    m_pDriver->HandleOOM(true);
+  }
+
+  ~ScopedOOMHandleVk() { m_pDriver->HandleOOM(false); }
+  WrappedVulkan *m_pDriver;
+};
+
 struct VkXfbQueryResult
 {
   uint64_t numPrimitivesWritten;
@@ -3350,6 +3362,9 @@ void VulkanReplay::InitPostVSBuffers(uint32_t eventId, VulkanRenderState state)
   if(m_PostVS.Data.find(eventId) != m_PostVS.Data.end())
     return;
 
+  // we handle out-of-memory errors while processing postvs, don't treat it as a fatal error
+  ScopedOOMHandleVk oom(m_pDriver);
+
   VulkanCreationInfo &creationInfo = m_pDriver->m_CreationInfo;
 
   if(state.graphics.pipeline == ResourceId() ||