Check for presence of extensions/layers, and platform surface extensions

* If one of the extensions or layers listed in the log isn't present on
  replay, we will just fail to load the capture. It's impossible to know
  whether or not it's safe to drop unsupported layers.
* Note that RenderDoc already suppresses reporting device extensions to
  the application if they're not supported by capture, which affects
  this.
* Also when adding platform surface extensions we check for availability
  and only add those present. We do require some minimum amount, but in
  future we'll need to report which windowing systems are supported to
  the UI layer so it can work appropriately using e.g. xcb or xlib.

renderdoc/driver/vulkan/vk_core.h

@@ -531,7 +531,8 @@ private:
   VulkanDrawcallTreeNode m_ParentDrawcall;
 
   // in vk_<platform>.cpp
-  void AddRequiredExtensions(bool instance, vector<string> &extensionList);
+  bool AddRequiredExtensions(bool instance, vector<string> &extensionList,
+                             const std::set<string> &supportedExtensions);
 
   void InsertDrawsAndRefreshIDs(vector<VulkanDrawcallTreeNode> &cmdBufNodes, uint32_t baseEventID,
                                 uint32_t baseDrawID);
@@ -590,7 +591,7 @@ public:
 
   bool ReleaseResource(WrappedVkRes *res);
 
-  void Initialise(VkInitParams &params);
+  ReplayCreateStatus Initialise(VkInitParams &params);
   void Shutdown();
   void ReplayLog(uint32_t startEventID, uint32_t endEventID, ReplayLogType replayType);
   void ReadLogInitialisation();

renderdoc/driver/vulkan/vk_dispatchtables.cpp

@@ -57,13 +57,15 @@ void InitReplayTables(void *vulkanModule)
     VkLayerInstanceDispatchTableExtended &table = replayInstanceTable;
     memset(&table, 0, sizeof(table));
     HookInit(GetInstanceProcAddr);
+    HookInit(EnumerateInstanceExtensionProperties);
+    HookInit(EnumerateInstanceLayerProperties);
     HookInitVulkanInstance();
   }
 }
 
 #define InstanceGPA(func) \
   table->func =           \
-      (CONCAT(PFN_vk, func))table->GetInstanceProcAddr(instance, STRINGIZE(CONCAT(vk, func)));
+      (CONCAT(PFN_vk, func))table->GetInstanceProcAddr(instance, STRINGIZE(CONCAT(vk, func)))
 
 void InitInstanceReplayTables(VkInstance instance)
 {
@@ -72,30 +74,34 @@ void InitInstanceReplayTables(VkInstance instance)
 
   // we know we'll only have one instance, so this is safe
 
-  InstanceGPA(EnumerateDeviceExtensionProperties) InstanceGPA(EnumerateDeviceLayerProperties)
+  InstanceGPA(EnumerateDeviceExtensionProperties);
+  InstanceGPA(EnumerateDeviceLayerProperties);
 
-      InstanceGPA(GetPhysicalDeviceSurfaceCapabilitiesKHR) InstanceGPA(
-          GetPhysicalDeviceSurfaceFormatsKHR) InstanceGPA(GetPhysicalDeviceSurfacePresentModesKHR)
-          InstanceGPA(GetPhysicalDeviceSurfaceSupportKHR) InstanceGPA(CreateDebugReportCallbackEXT)
-              InstanceGPA(DestroyDebugReportCallbackEXT) InstanceGPA(DebugReportMessageEXT)
+  InstanceGPA(GetPhysicalDeviceSurfaceCapabilitiesKHR);
+  InstanceGPA(GetPhysicalDeviceSurfaceFormatsKHR);
+  InstanceGPA(GetPhysicalDeviceSurfacePresentModesKHR);
+  InstanceGPA(GetPhysicalDeviceSurfaceSupportKHR);
+  InstanceGPA(CreateDebugReportCallbackEXT);
+  InstanceGPA(DestroyDebugReportCallbackEXT);
+  InstanceGPA(DebugReportMessageEXT);
 
 #ifdef VK_USE_PLATFORM_WIN32_KHR
-                  InstanceGPA(CreateWin32SurfaceKHR)
+  InstanceGPA(CreateWin32SurfaceKHR);
 #endif
 
 #ifdef VK_USE_PLATFORM_ANDROID_KHR
-                      InstanceGPA(CreateAndroidSurfaceKHR)
+  InstanceGPA(CreateAndroidSurfaceKHR);
 #endif
 
 #ifdef VK_USE_PLATFORM_XCB_KHR
-                          InstanceGPA(CreateXcbSurfaceKHR)
+  InstanceGPA(CreateXcbSurfaceKHR);
 #endif
 
 #ifdef VK_USE_PLATFORM_XLIB_KHR
-                              InstanceGPA(CreateXlibSurfaceKHR)
+  InstanceGPA(CreateXlibSurfaceKHR);
 #endif
 
-                                  InstanceGPA(DestroySurfaceKHR)
+  InstanceGPA(DestroySurfaceKHR);
 }
 
 void InitInstanceExtensionTables(VkInstance instance)

renderdoc/driver/vulkan/vk_hookset_defs.h

@@ -605,6 +605,10 @@ struct VkLayerInstanceDispatchTableExtended : VkLayerInstanceDispatchTable
   // even though it won't actually ever get used
   PFN_vkCreateInstance CreateInstance;
 
+  // we need to use these before we have a dispatch table
+  PFN_vkEnumerateInstanceExtensionProperties EnumerateInstanceExtensionProperties;
+  PFN_vkEnumerateInstanceLayerProperties EnumerateInstanceLayerProperties;
+
   // extensions here
 };
 

renderdoc/driver/vulkan/vk_posix.cpp

@@ -35,28 +35,68 @@ bool VulkanReplay::IsOutputWindowVisible(uint64_t id)
   return true;
 }
 
-void WrappedVulkan::AddRequiredExtensions(bool instance, vector<string> &extensionList)
+bool WrappedVulkan::AddRequiredExtensions(bool instance, vector<string> &extensionList,
+                                          const std::set<string> &supportedExtensions)
 {
   bool device = !instance;
 
-  // TODO should check if these are present..
-
   if(instance)
   {
-    extensionList.push_back(VK_KHR_SURFACE_EXTENSION_NAME);
+    // we must have VK_KHR_surface
+    if(supportedExtensions.find(VK_KHR_SURFACE_EXTENSION_NAME) == supportedExtensions.end())
+    {
+      RDCERR("Unsupported required instance extension '%s'", VK_KHR_SURFACE_EXTENSION_NAME);
+      return false;
+    }
+
+    // don't add duplicates
+    if(std::find(extensionList.begin(), extensionList.end(), VK_KHR_SURFACE_EXTENSION_NAME) ==
+       extensionList.end())
+      extensionList.push_back(VK_KHR_SURFACE_EXTENSION_NAME);
+
+    bool oneSurfaceTypeSupported = false;
 
 #if defined(VK_USE_PLATFORM_XCB_KHR)
-    extensionList.push_back(VK_KHR_XCB_SURFACE_EXTENSION_NAME);
+    // don't add duplicates, and only add if supported
+    if(supportedExtensions.find(VK_KHR_XCB_SURFACE_EXTENSION_NAME) != supportedExtensions.end() &&
+       std::find(extensionList.begin(), extensionList.end(), VK_KHR_XCB_SURFACE_EXTENSION_NAME) ==
+           extensionList.end())
+    {
+      extensionList.push_back(VK_KHR_XCB_SURFACE_EXTENSION_NAME);
+      oneSurfaceTypeSupported = true;
+    }
 #endif
 
 #if defined(VK_USE_PLATFORM_XLIB_KHR)
-    extensionList.push_back(VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
+    // don't add duplicates
+    if(supportedExtensions.find(VK_KHR_XLIB_SURFACE_EXTENSION_NAME) != supportedExtensions.end() &&
+       std::find(extensionList.begin(), extensionList.end(), VK_KHR_XLIB_SURFACE_EXTENSION_NAME) ==
+           extensionList.end())
+    {
+      extensionList.push_back(VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
+      oneSurfaceTypeSupported = true;
+    }
 #endif
+
+    if(!oneSurfaceTypeSupported)
+    {
+      RDCERR("Required at least one of '%s' or '%s' extension to be present",
+             VK_KHR_XCB_SURFACE_EXTENSION_NAME, VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
+      return false;
+    }
   }
   else if(device)
   {
+    if(supportedExtensions.find(VK_KHR_SWAPCHAIN_EXTENSION_NAME) == supportedExtensions.end())
+    {
+      RDCERR("Unsupported required device extension '%s'", VK_KHR_SWAPCHAIN_EXTENSION_NAME);
+      return false;
+    }
+
     extensionList.push_back(VK_KHR_SWAPCHAIN_EXTENSION_NAME);
   }
+
+  return true;
 }
 
 #if defined(VK_USE_PLATFORM_XCB_KHR)

renderdoc/driver/vulkan/vk_replay.cpp

@@ -5304,7 +5304,13 @@ ReplayCreateStatus Vulkan_CreateReplayDevice(const char *logfile, IReplayDriver
   VulkanReplay::PreDeviceInitCounters();
 
   WrappedVulkan *vk = new WrappedVulkan(logfile);
-  vk->Initialise(initParams);
+  ReplayCreateStatus status = vk->Initialise(initParams);
+
+  if(status != eReplayCreate_Success)
+  {
+    delete vk;
+    return status;
+  }
 
   RDCLOG("Created device.");
   VulkanReplay *replay = vk->GetReplay();

renderdoc/driver/vulkan/vk_win32.cpp

@@ -70,21 +70,50 @@ bool VulkanReplay::IsOutputWindowVisible(uint64_t id)
   return (IsWindowVisible(m_OutputWindows[id].wnd) == TRUE);
 }
 
-void WrappedVulkan::AddRequiredExtensions(bool instance, vector<string> &extensionList)
+bool WrappedVulkan::AddRequiredExtensions(bool instance, vector<string> &extensionList,
+                                          const std::set<string> &supportedExtensions)
 {
   bool device = !instance;
 
-  // TODO should check if these are present..
-
   if(instance)
   {
-    extensionList.push_back(VK_KHR_SURFACE_EXTENSION_NAME);
-    extensionList.push_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME);
+    // for windows we require both extensions as there's no alternative
+    if(supportedExtensions.find(VK_KHR_SURFACE_EXTENSION_NAME) == supportedExtensions.end())
+    {
+      RDCERR("Unsupported required instance extension '%s'", VK_KHR_SURFACE_EXTENSION_NAME);
+      return false;
+    }
+
+    if(supportedExtensions.find(VK_KHR_WIN32_SURFACE_EXTENSION_NAME) == supportedExtensions.end())
+    {
+      RDCERR("Unsupported required instance extension '%s'", VK_KHR_WIN32_SURFACE_EXTENSION_NAME);
+      return false;
+    }
+
+    // don't add duplicates
+    if(std::find(extensionList.begin(), extensionList.end(), VK_KHR_SURFACE_EXTENSION_NAME) ==
+       extensionList.end())
+      extensionList.push_back(VK_KHR_SURFACE_EXTENSION_NAME);
+
+    if(std::find(extensionList.begin(), extensionList.end(), VK_KHR_WIN32_SURFACE_EXTENSION_NAME) ==
+       extensionList.end())
+      extensionList.push_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME);
   }
   else if(device)
   {
-    extensionList.push_back(VK_KHR_SWAPCHAIN_EXTENSION_NAME);
+    if(supportedExtensions.find(VK_KHR_SWAPCHAIN_EXTENSION_NAME) == supportedExtensions.end())
+    {
+      RDCERR("Unsupported required device extension '%s'", VK_KHR_SWAPCHAIN_EXTENSION_NAME);
+      return false;
+    }
+
+    // don't add duplicates
+    if(std::find(extensionList.begin(), extensionList.end(), VK_KHR_SWAPCHAIN_EXTENSION_NAME) ==
+       extensionList.end())
+      extensionList.push_back(VK_KHR_SWAPCHAIN_EXTENSION_NAME);
   }
+
+  return true;
 }
 
 #if !defined(VK_USE_PLATFORM_WIN32_KHR)

renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp

@@ -82,7 +82,7 @@ static void StripUnwantedLayers(vector<string> &Layers)
   }
 }
 
-void WrappedVulkan::Initialise(VkInitParams &params)
+ReplayCreateStatus WrappedVulkan::Initialise(VkInitParams &params)
 {
   m_InitParams = params;
 
@@ -115,7 +115,63 @@ void WrappedVulkan::Initialise(VkInitParams &params)
     }
   }
 
-  AddRequiredExtensions(true, params.Extensions);
+  std::set<string> supportedExtensions;
+
+  for(size_t i = 0; i <= params.Layers.size(); i++)
+  {
+    const char *pLayerName = (i == 0 ? NULL : params.Layers[i - 1].c_str());
+
+    uint32_t count = 0;
+    GetInstanceDispatchTable(NULL)->EnumerateInstanceExtensionProperties(pLayerName, &count, NULL);
+
+    VkExtensionProperties *props = new VkExtensionProperties[count];
+    GetInstanceDispatchTable(NULL)->EnumerateInstanceExtensionProperties(pLayerName, &count, props);
+
+    for(uint32_t e = 0; e < count; e++)
+      supportedExtensions.insert(props[e].extensionName);
+
+    SAFE_DELETE_ARRAY(props);
+  }
+
+  std::set<string> supportedLayers;
+
+  {
+    uint32_t count = 0;
+    GetInstanceDispatchTable(NULL)->EnumerateInstanceLayerProperties(&count, NULL);
+
+    VkLayerProperties *props = new VkLayerProperties[count];
+    GetInstanceDispatchTable(NULL)->EnumerateInstanceLayerProperties(&count, props);
+
+    for(uint32_t e = 0; e < count; e++)
+      supportedLayers.insert(props[e].layerName);
+
+    SAFE_DELETE_ARRAY(props);
+  }
+
+  bool ok = AddRequiredExtensions(true, params.Extensions, supportedExtensions);
+
+  // error message will be printed to log in above function if something went wrong
+  if(!ok)
+    return eReplayCreate_APIHardwareUnsupported;
+
+  // verify that extensions & layers are supported
+  for(size_t i = 0; i < params.Layers.size(); i++)
+  {
+    if(supportedLayers.find(params.Layers[i]) == supportedLayers.end())
+    {
+      RDCERR("Log requires layer '%s' which is not supported", params.Layers[i].c_str());
+      return eReplayCreate_APIHardwareUnsupported;
+    }
+  }
+
+  for(size_t i = 0; i < params.Extensions.size(); i++)
+  {
+    if(supportedExtensions.find(params.Extensions[i]) == supportedExtensions.end())
+    {
+      RDCERR("Log requires extension '%s' which is not supported", params.Extensions[i].c_str());
+      return eReplayCreate_APIHardwareUnsupported;
+    }
+  }
 
   const char **layerscstr = new const char *[params.Layers.size()];
   for(size_t i = 0; i < params.Layers.size(); i++)
@@ -149,6 +205,13 @@ void WrappedVulkan::Initialise(VkInitParams &params)
   m_Instance = VK_NULL_HANDLE;
 
   VkResult ret = GetInstanceDispatchTable(NULL)->CreateInstance(&instinfo, NULL, &m_Instance);
+
+  SAFE_DELETE_ARRAY(layerscstr);
+  SAFE_DELETE_ARRAY(extscstr);
+
+  if(ret != VK_SUCCESS)
+    return eReplayCreate_APIHardwareUnsupported;
+
   RDCASSERTEQUAL(ret, VK_SUCCESS);
 
   InitInstanceReplayTables(m_Instance);
@@ -177,8 +240,7 @@ void WrappedVulkan::Initialise(VkInitParams &params)
         ->CreateDebugReportCallbackEXT(Unwrap(m_Instance), &debugInfo, NULL, &m_DbgMsgCallback);
   }
 
-  SAFE_DELETE_ARRAY(layerscstr);
-  SAFE_DELETE_ARRAY(extscstr);
+  return eReplayCreate_Success;
 }
 
 VkResult WrappedVulkan::vkCreateInstance(const VkInstanceCreateInfo *pCreateInfo,
@@ -612,7 +674,29 @@ bool WrappedVulkan::Serialise_vkCreateDevice(Serialiser *localSerialiser,
 
     StripUnwantedLayers(Layers);
 
-    AddRequiredExtensions(false, Extensions);
+    physicalDevice = GetResourceManager()->GetLiveHandle<VkPhysicalDevice>(physId);
+
+    std::set<string> supportedExtensions;
+
+    for(size_t i = 0; i <= Layers.size(); i++)
+    {
+      const char *pLayerName = (i == 0 ? NULL : Layers[i - 1].c_str());
+
+      uint32_t count = 0;
+      ObjDisp(physicalDevice)
+          ->EnumerateDeviceExtensionProperties(Unwrap(physicalDevice), pLayerName, &count, NULL);
+
+      VkExtensionProperties *props = new VkExtensionProperties[count];
+      ObjDisp(physicalDevice)
+          ->EnumerateDeviceExtensionProperties(Unwrap(physicalDevice), pLayerName, &count, props);
+
+      for(uint32_t e = 0; e < count; e++)
+        supportedExtensions.insert(props[e].extensionName);
+
+      SAFE_DELETE_ARRAY(props);
+    }
+
+    AddRequiredExtensions(false, Extensions, supportedExtensions);
 
 #if defined(FORCE_VALIDATION_LAYERS)
     Layers.push_back("VK_LAYER_LUNARG_standard_validation");
@@ -644,8 +728,6 @@ bool WrappedVulkan::Serialise_vkCreateDevice(Serialiser *localSerialiser,
       createInfo.ppEnabledExtensionNames = extArray;
     }
 
-    physicalDevice = GetResourceManager()->GetLiveHandle<VkPhysicalDevice>(physId);
-
     VkDevice device;
 
     uint32_t qCount = 0;