If the unique_objects layer is enabled, try to replace objects with IDs

* This isn't perfect and has the risk of false positives, e.g. 0x10 for a
  bitfield might be identified as the object with handle 16. Without doing more
  advanced string processing like trying to identify 'image 0x10' vs
  'aspectFlags 0x10' this is unavoidable, and adding a nonsense ResourceId where
  context makes it clear that it's nonsense isn't too bad.
* If in future the validation layers are updated to consistently pass objects
  through the debug_utils interface, this may become redundant.

renderdoc/driver/vulkan/vk_common.h

@@ -219,6 +219,12 @@ enum
   VkCheckExt_Max,
 };
 
+enum
+{
+  VkCheckLayer_unique_objects,
+  VkCheckLayer_Max,
+};
+
 DECLARE_REFLECTION_STRUCT(VkBaseInStructure);
 
 // we cast to this type when serialising as a placeholder indicating that

renderdoc/driver/vulkan/vk_core.cpp

@@ -2946,6 +2946,76 @@ void WrappedVulkan::Serialise_DebugMessages(SerialiserType &ser)
     ScopedDebugMessageSink *sink = GetDebugMessageSink();
     if(sink)
       DebugMessages.swap(sink->msgs);
+
+    // if we have the unique objects layer we can assume all objects have a unique ID, and replace
+    // any text that looks like an object reference (0xHEX).
+    if(m_LayersEnabled[VkCheckLayer_unique_objects])
+    {
+      for(DebugMessage &msg : DebugMessages)
+      {
+        if(strstr(msg.description.c_str(), "0x"))
+        {
+          std::string desc = msg.description;
+
+          size_t offs = desc.find("0x");
+          while(offs != std::string::npos)
+          {
+            // if we're on a word boundary
+            if(offs == 0 || !isalnum(desc[offs - 1]))
+            {
+              size_t end = offs + 2;
+
+              uint64_t val = 0;
+
+              // consume all hex chars
+              while(end < desc.length())
+              {
+                if(desc[end] >= '0' && desc[end] <= '9')
+                {
+                  val <<= 4;
+                  val += (desc[end] - '0');
+                  end++;
+                }
+                else if(desc[end] >= 'A' && desc[end] <= 'F')
+                {
+                  val <<= 4;
+                  val += (desc[end] - 'A') + 0xA;
+                  end++;
+                }
+                else if(desc[end] >= 'a' && desc[end] <= 'f')
+                {
+                  val <<= 4;
+                  val += (desc[end] - 'a') + 0xA;
+                  end++;
+                }
+                else
+                {
+                  break;
+                }
+              }
+
+              // unique objects layer implies this is a unique search so we don't have to worry
+              // about type aliases
+              ResourceId id = GetResourceManager()->GetFirstIDForHandle(val);
+
+              if(id != ResourceId())
+              {
+                std::string idstr = StringFormat::Fmt(" (%s)", ToStr(id).c_str());
+
+                desc.insert(end, idstr.c_str());
+
+                offs = desc.find("0x", end + idstr.length());
+                continue;
+              }
+            }
+
+            offs = desc.find("0x", offs + 1);
+          }
+
+          msg.description = desc;
+        }
+      }
+    }
   }
 
   SERIALISE_ELEMENT(DebugMessages);

renderdoc/driver/vulkan/vk_core.h

@@ -815,7 +815,8 @@ private:
 
   VulkanDrawcallTreeNode m_ParentDrawcall;
 
-  bool m_ExtensionsEnabled[VkCheckExt_Max];
+  bool m_ExtensionsEnabled[VkCheckExt_Max] = {};
+  bool m_LayersEnabled[VkCheckLayer_Max] = {};
 
   // in vk_<platform>.cpp
   void AddRequiredExtensions(bool instance, vector<string> &extensionList,

renderdoc/driver/vulkan/vk_manager.cpp

@@ -576,6 +576,32 @@ void VulkanResourceManager::ApplyBarriers(uint32_t queueFamilyIndex,
   }
 }
 
+ResourceId VulkanResourceManager::GetFirstIDForHandle(uint64_t handle)
+{
+  for(auto it = m_ResourceRecords.begin(); it != m_ResourceRecords.end(); ++it)
+  {
+    WrappedVkRes *res = it->second->Resource;
+
+    if(!res)
+      continue;
+
+    if(IsDispatchableRes(res))
+    {
+      WrappedVkDispRes *disp = (WrappedVkDispRes *)res;
+      if(disp->real.handle == handle)
+        return disp->id;
+    }
+    else
+    {
+      WrappedVkNonDispRes *nondisp = (WrappedVkNonDispRes *)res;
+      if(nondisp->real.handle == handle)
+        return nondisp->id;
+    }
+  }
+
+  return ResourceId();
+}
+
 bool VulkanResourceManager::Force_InitialState(WrappedVkRes *res, bool prepare)
 {
   return false;

renderdoc/driver/vulkan/vk_manager.h

@@ -223,6 +223,8 @@ public:
     return ret;
   }
 
+  ResourceId GetFirstIDForHandle(uint64_t handle);
+
   // easy path for getting the unwrapped handle cast to the
   // write type. Saves a lot of work casting to either WrappedVkNonDispRes
   // or WrappedVkDispRes depending on the type, then ->real, then casting

renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp

@@ -165,8 +165,6 @@ ReplayStatus WrappedVulkan::Initialise(VkInitParams &params, uint64_t sectionVer
     }
   }
 
-  RDCEraseEl(m_ExtensionsEnabled);
-
   std::set<string> supportedExtensions;
 
   for(size_t i = 0; i <= params.Layers.size(); i++)
@@ -489,6 +487,18 @@ VkResult WrappedVulkan::vkCreateInstance(const VkInstanceCreateInfo *pCreateInfo
     modifiedCreateInfo.pApplicationInfo = &renderdocAppInfo;
   }
 
+  for(uint32_t i = 0; i < modifiedCreateInfo.enabledLayerCount; i++)
+  {
+    if(!strcmp(modifiedCreateInfo.ppEnabledLayerNames[i], "VK_LAYER_LUNARG_standard_validation") ||
+       !strcmp(modifiedCreateInfo.ppEnabledLayerNames[i], "VK_LAYER_GOOGLE_unique_objects"))
+    {
+      m_LayersEnabled[VkCheckLayer_unique_objects] = true;
+    }
+  }
+
+  // if we forced on API validation, it's also available
+  m_LayersEnabled[VkCheckLayer_unique_objects] |= RenderDoc::Inst().GetCaptureOptions().apiValidation;
+
   VkResult ret = createFunc(&modifiedCreateInfo, pAllocator, pInstance);
 
   m_Instance = *pInstance;