diff --git a/renderdoc/driver/d3d11/d3d11_device.cpp b/renderdoc/driver/d3d11/d3d11_device.cpp index 4b54f55ee..4500afcb1 100644 --- a/renderdoc/driver/d3d11/d3d11_device.cpp +++ b/renderdoc/driver/d3d11/d3d11_device.cpp @@ -2535,8 +2535,15 @@ void WrappedID3D11Device::CheckHRESULT(HRESULT hr) } else if(hr == E_OUTOFMEMORY) { - RDCLOG("Logging out of memory fatal error for %s", ToStr(hr).c_str()); - m_FatalError = ReplayStatus::ReplayOutOfMemory; + if(m_OOMHandler) + { + RDCLOG("Ignoring out of memory error that will be handled"); + } + else + { + RDCLOG("Logging out of memory fatal error for %s", ToStr(hr).c_str()); + m_FatalError = ReplayStatus::ReplayOutOfMemory; + } } else { diff --git a/renderdoc/driver/d3d11/d3d11_device.h b/renderdoc/driver/d3d11/d3d11_device.h index 4437e0e2d..618afadb5 100644 --- a/renderdoc/driver/d3d11/d3d11_device.h +++ b/renderdoc/driver/d3d11/d3d11_device.h @@ -600,6 +600,7 @@ private: SDFile *m_StructuredFile = NULL; SDFile *m_StoredStructuredData; + int m_OOMHandler = 0; rdcarray m_DebugMessages; ReplayStatus m_FatalError = ReplayStatus::Succeeded; @@ -673,6 +674,13 @@ public: double GetTimeFrequency() { return m_TimeFrequency; } void FirstFrame(IDXGISwapper *swapper); + void HandleOOM(bool handle) + { + if(handle) + m_OOMHandler++; + else + m_OOMHandler--; + } void CheckHRESULT(HRESULT hr); void ReportFatalError(ReplayStatus error) { m_FatalError = error; } ReplayStatus FatalErrorCheck() { return m_FatalError; } diff --git a/renderdoc/driver/d3d11/d3d11_postvs.cpp b/renderdoc/driver/d3d11/d3d11_postvs.cpp index d3b3da86d..54128be10 100644 --- a/renderdoc/driver/d3d11/d3d11_postvs.cpp +++ b/renderdoc/driver/d3d11/d3d11_postvs.cpp @@ -32,6 +32,18 @@ #include "d3d11_replay.h" #include "d3d11_resources.h" +struct ScopedOOMHandle11 +{ + ScopedOOMHandle11(WrappedID3D11Device *dev) + { + m_pDevice = dev; + m_pDevice->HandleOOM(true); + } + + ~ScopedOOMHandle11() { m_pDevice->HandleOOM(false); } + WrappedID3D11Device *m_pDevice; +}; + void D3D11Replay::InitStreamOut() { CreateSOBuffers(); @@ -179,6 +191,9 @@ void D3D11Replay::InitPostVSBuffers(uint32_t eventId) if(m_PostVSData.find(eventId) != m_PostVSData.end()) return; + // we handle out-of-memory errors while processing postvs, don't treat it as a fatal error + ScopedOOMHandle11 oom(m_pDevice); + D3D11MarkerRegion postvs(StringFormat::Fmt("PostVS for %u", eventId)); D3D11RenderStateTracker tracker(m_pImmediateContext); diff --git a/renderdoc/driver/d3d12/d3d12_device.cpp b/renderdoc/driver/d3d12/d3d12_device.cpp index 7a33ccb28..07f1c48fd 100644 --- a/renderdoc/driver/d3d12/d3d12_device.cpp +++ b/renderdoc/driver/d3d12/d3d12_device.cpp @@ -3022,8 +3022,15 @@ void WrappedID3D12Device::CheckHRESULT(HRESULT hr) } else if(hr == E_OUTOFMEMORY) { - RDCLOG("Logging out of memory fatal error for %s", ToStr(hr).c_str()); - m_FatalError = ReplayStatus::ReplayOutOfMemory; + if(m_OOMHandler) + { + RDCLOG("Ignoring out of memory error that will be handled"); + } + else + { + RDCLOG("Logging out of memory fatal error for %s", ToStr(hr).c_str()); + m_FatalError = ReplayStatus::ReplayOutOfMemory; + } } else { diff --git a/renderdoc/driver/d3d12/d3d12_device.h b/renderdoc/driver/d3d12/d3d12_device.h index fb424070e..e03ffe17b 100644 --- a/renderdoc/driver/d3d12/d3d12_device.h +++ b/renderdoc/driver/d3d12/d3d12_device.h @@ -667,6 +667,7 @@ private: rdcarray m_ThreadTempMem; rdcarray m_DebugMessages; + int m_OOMHandler = 0; ReplayStatus m_FatalError = ReplayStatus::Succeeded; uint64_t m_TimeBase = 0; @@ -801,6 +802,13 @@ public: void AddDebugMessage(const DebugMessage &msg); rdcarray GetDebugMessages(); + void HandleOOM(bool handle) + { + if(handle) + m_OOMHandler++; + else + m_OOMHandler--; + } void CheckHRESULT(HRESULT hr); void ReportFatalError(ReplayStatus error) { m_FatalError = error; } ReplayStatus FatalErrorCheck() { return m_FatalError; } diff --git a/renderdoc/driver/d3d12/d3d12_postvs.cpp b/renderdoc/driver/d3d12/d3d12_postvs.cpp index 990dc7a7f..255bb55c8 100644 --- a/renderdoc/driver/d3d12/d3d12_postvs.cpp +++ b/renderdoc/driver/d3d12/d3d12_postvs.cpp @@ -32,6 +32,18 @@ #include "d3d12_replay.h" #include "d3d12_shader_cache.h" +struct ScopedOOMHandle12 +{ + ScopedOOMHandle12(WrappedID3D12Device *dev) + { + m_pDevice = dev; + m_pDevice->HandleOOM(true); + } + + ~ScopedOOMHandle12() { m_pDevice->HandleOOM(false); } + WrappedID3D12Device *m_pDevice; +}; + bool D3D12Replay::CreateSOBuffers() { HRESULT hr = S_OK; @@ -170,6 +182,9 @@ void D3D12Replay::InitPostVSBuffers(uint32_t eventId) if(m_PostVSData.find(eventId) != m_PostVSData.end()) return; + // we handle out-of-memory errors while processing postvs, don't treat it as a fatal error + ScopedOOMHandle12 oom(m_pDevice); + D3D12MarkerRegion postvs(m_pDevice->GetQueue(), StringFormat::Fmt("PostVS for %u", eventId)); D3D12CommandData *cmd = m_pDevice->GetQueue()->GetCommandData(); diff --git a/renderdoc/driver/vulkan/vk_core.cpp b/renderdoc/driver/vulkan/vk_core.cpp index 43bba5120..00db7f2a7 100644 --- a/renderdoc/driver/vulkan/vk_core.cpp +++ b/renderdoc/driver/vulkan/vk_core.cpp @@ -4009,8 +4009,15 @@ void WrappedVulkan::CheckErrorVkResult(VkResult vkr) } else if(vkr == VK_ERROR_OUT_OF_HOST_MEMORY || vkr == VK_ERROR_OUT_OF_DEVICE_MEMORY) { - RDCLOG("Logging out of memory fatal error for %s", ToStr(vkr).c_str()); - m_FailedReplayStatus = m_FatalError = ReplayStatus::ReplayOutOfMemory; + if(m_OOMHandler) + { + RDCLOG("Ignoring out of memory error that will be handled"); + } + else + { + RDCLOG("Logging out of memory fatal error for %s", ToStr(vkr).c_str()); + m_FailedReplayStatus = m_FatalError = ReplayStatus::ReplayOutOfMemory; + } } else { diff --git a/renderdoc/driver/vulkan/vk_core.h b/renderdoc/driver/vulkan/vk_core.h index e0780e204..d8f953cc5 100644 --- a/renderdoc/driver/vulkan/vk_core.h +++ b/renderdoc/driver/vulkan/vk_core.h @@ -289,6 +289,7 @@ private: rdcarray GetDebugMessages(); void AddDebugMessage(DebugMessage msg); + int m_OOMHandler = 0; ReplayStatus m_FatalError = ReplayStatus::Succeeded; CaptureState m_State; bool m_AppControlledCapture = false; @@ -1110,6 +1111,13 @@ public: bool SelectGraphicsComputeQueue(const rdcarray &queueProps, VkDeviceCreateInfo &createInfo, uint32_t &queueFamilyIndex); + void HandleOOM(bool handle) + { + if(handle) + m_OOMHandler++; + else + m_OOMHandler--; + } ReplayStatus FatalErrorCheck() { return m_FatalError; } bool HasFatalError() { return m_FatalError != ReplayStatus::Succeeded; } inline void CheckVkResult(VkResult vkr) diff --git a/renderdoc/driver/vulkan/vk_postvs.cpp b/renderdoc/driver/vulkan/vk_postvs.cpp index 746059189..7273ae770 100644 --- a/renderdoc/driver/vulkan/vk_postvs.cpp +++ b/renderdoc/driver/vulkan/vk_postvs.cpp @@ -39,6 +39,18 @@ RDOC_EXTERN_CONFIG(bool, Vulkan_Debug_DisableBufferDeviceAddress); #undef None +struct ScopedOOMHandleVk +{ + ScopedOOMHandleVk(WrappedVulkan *vk) + { + m_pDriver = vk; + m_pDriver->HandleOOM(true); + } + + ~ScopedOOMHandleVk() { m_pDriver->HandleOOM(false); } + WrappedVulkan *m_pDriver; +}; + struct VkXfbQueryResult { uint64_t numPrimitivesWritten; @@ -3350,6 +3362,9 @@ void VulkanReplay::InitPostVSBuffers(uint32_t eventId, VulkanRenderState state) if(m_PostVS.Data.find(eventId) != m_PostVS.Data.end()) return; + // we handle out-of-memory errors while processing postvs, don't treat it as a fatal error + ScopedOOMHandleVk oom(m_pDriver); + VulkanCreationInfo &creationInfo = m_pDriver->m_CreationInfo; if(state.graphics.pipeline == ResourceId() ||