Added GetMaxMemoryAllocationSize() to Vulkan device

For use during replay to enable more situations for graceful handling of OOM
VkAllocateMemory can return VK_UNKNOWN_ERROR in some situations
Used in FetchVSOut() to gracefully handle draws with very large vertex counts

renderdoc/driver/vulkan/vk_core.h

@@ -437,6 +437,8 @@ private:
     VkDriverInfo driverInfo = VkDriverInfo(false);
 
     VkPhysicalDevicePerformanceQueryFeaturesKHR performanceQueryFeatures = {};
+    // Only set during replay
+    VkDeviceSize maxMemoryAllocationSize = 0U;
 
     uint32_t queueCount = 0;
     VkQueueFamilyProperties queueProps[16] = {};
@@ -1305,6 +1307,10 @@ public:
   {
     return m_PhysicalDeviceData.performanceQueryFeatures;
   }
+  const VkDeviceSize &GetMaxMemoryAllocationSize()
+  {
+    return m_PhysicalDeviceData.maxMemoryAllocationSize;
+  }
   const VkDriverInfo &GetDriverInfo() const { return m_PhysicalDeviceData.driverInfo; }
   uint32_t FindCommandQueueFamily(ResourceId cmdId);
   void InsertCommandQueueFamily(ResourceId cmdId, uint32_t queueFamilyIndex);

renderdoc/driver/vulkan/vk_postvs.cpp

@@ -5070,6 +5070,13 @@ void VulkanReplay::FetchVSOut(uint32_t eventId, VulkanRenderState &state)
     VkMemoryRequirements mrq = {0};
     m_pDriver->vkGetBufferMemoryRequirements(dev, meshBuffer, &mrq);
 
+    if(mrq.size > m_pDriver->GetMaxMemoryAllocationSize())
+    {
+      ret.vsout.status = StringFormat::Fmt("OOM %llu bytes Max %llu bytes", mrq.size,
+                                           m_pDriver->GetMaxMemoryAllocationSize());
+      return;
+    }
+
     VkMemoryAllocateInfo allocInfo = {
         VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
         NULL,

renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp

@@ -4048,6 +4048,22 @@ bool WrappedVulkan::Serialise_vkCreateDevice(SerialiserType &ser, VkPhysicalDevi
     for(size_t i = 0; i < queueProps.size(); i++)
       m_PhysicalDeviceData.queueProps[i] = queueProps[i];
 
+    if(RDCMIN(m_EnabledExtensions.vulkanVersion, physProps.apiVersion) >= VK_MAKE_VERSION(1, 1, 0))
+    {
+      VkPhysicalDeviceVulkan11Properties vulkan11Props = {
+          VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES,
+      };
+
+      VkPhysicalDeviceProperties2 devProps2 = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2};
+      devProps2.pNext = &vulkan11Props;
+      ObjDisp(physicalDevice)->GetPhysicalDeviceProperties2(Unwrap(physicalDevice), &devProps2);
+      m_PhysicalDeviceData.maxMemoryAllocationSize = vulkan11Props.maxMemoryAllocationSize;
+    }
+    else
+    {
+      m_PhysicalDeviceData.maxMemoryAllocationSize = 0x80000000U;
+    }
+
     ChooseMemoryIndices();
 
     APIProps.vendor = GetDriverInfo().Vendor();
@@ -4512,6 +4528,7 @@ VkResult WrappedVulkan::vkCreateDevice(VkPhysicalDevice physicalDevice,
       RDCLOG("Forcing 200MB soft memory limit");
     }
 
+    m_PhysicalDeviceData.maxMemoryAllocationSize = 0;
     ChooseMemoryIndices();
 
     m_PhysicalDeviceData.queueCount = (uint32_t)queueProps.size();