diff --git a/renderdoc/driver/vulkan/vk_common.cpp b/renderdoc/driver/vulkan/vk_common.cpp
index 15e64d3f7..291a67094 100644
--- a/renderdoc/driver/vulkan/vk_common.cpp
+++ b/renderdoc/driver/vulkan/vk_common.cpp
@@ -1057,41 +1057,44 @@ void DescriptorSetSlotImageInfo::SetFrom(const VkDescriptorImageInfo &imInfo, bo
   imageLayout = imInfo.imageLayout;
 }
 
-void DescriptorSetSlot::RemoveBindRefs(VulkanResourceManager *rm, VkResourceRecord *record)
+void DescriptorSetSlot::RemoveBindRefs(std::set<ResourceId> &ids, VulkanResourceManager *rm,
+                                       VkResourceRecord *record)
 {
   SCOPED_LOCK(record->descInfo->refLock);
 
   if(texelBufferView != ResourceId())
   {
-    record->RemoveBindFrameRef(texelBufferView);
+    record->RemoveBindFrameRef(ids, texelBufferView);
 
     VkResourceRecord *viewRecord = rm->GetResourceRecord(texelBufferView);
     if(viewRecord && viewRecord->baseResource != ResourceId())
-      record->RemoveBindFrameRef(viewRecord->baseResource);
+      record->RemoveBindFrameRef(ids, viewRecord->baseResource);
+    if(viewRecord && viewRecord->baseResourceMem != ResourceId())
+      record->RemoveBindFrameRef(ids, viewRecord->baseResourceMem);
   }
   if(imageInfo.imageView != ResourceId())
   {
-    record->RemoveBindFrameRef(imageInfo.imageView);
+    record->RemoveBindFrameRef(ids, imageInfo.imageView);
 
     VkResourceRecord *viewRecord = rm->GetResourceRecord(imageInfo.imageView);
     if(viewRecord)
     {
-      record->RemoveBindFrameRef(viewRecord->baseResource);
+      record->RemoveBindFrameRef(ids, viewRecord->baseResource);
       if(viewRecord->baseResourceMem != ResourceId())
-        record->RemoveBindFrameRef(viewRecord->baseResourceMem);
+        record->RemoveBindFrameRef(ids, viewRecord->baseResourceMem);
     }
   }
   if(imageInfo.sampler != ResourceId())
   {
-    record->RemoveBindFrameRef(imageInfo.sampler);
+    record->RemoveBindFrameRef(ids, imageInfo.sampler);
   }
   if(bufferInfo.buffer != ResourceId())
   {
-    record->RemoveBindFrameRef(bufferInfo.buffer);
+    record->RemoveBindFrameRef(ids, bufferInfo.buffer);
 
     VkResourceRecord *bufRecord = rm->GetResourceRecord(bufferInfo.buffer);
     if(bufRecord && bufRecord->baseResource != ResourceId())
-      record->RemoveBindFrameRef(bufRecord->baseResource);
+      record->RemoveBindFrameRef(ids, bufRecord->baseResource);
   }
 
   // NULL everything out now so that we don't accidentally reference an object
@@ -1146,7 +1149,6 @@ void DescriptorSetData::UpdateBackgroundRefCache(VulkanResourceManager *resource
   for(auto refit = ids.begin(); refit != ids.end(); ++refit)
   {
     ResourceId id = *refit;
-    FrameRefType refType = bindFrameRefs[id].second;
 
     // find the Id we're looking for in the remainder of the cache. This won't skip over any one
     // that we care about because we're iterating in ascending Id order
@@ -1156,6 +1158,18 @@ void DescriptorSetData::UpdateBackgroundRefCache(VulkanResourceManager *resource
           return a.first < b.first;
         });
 
+    auto bindit = bindFrameRefs.find(id);
+
+    // this id is no longer referenced, remove from the cache
+    if(bindit == bindFrameRefs.end())
+    {
+      if(cacheit != backgroundFrameRefs.end())
+        backgroundFrameRefs.erase(cacheit - backgroundFrameRefs.begin());
+      continue;
+    }
+
+    FrameRefType refType = bindit->second.second;
+
     // if we didn't find a match, insert the desired entry here
     if(cacheit == backgroundFrameRefs.end() || cacheit->first != id)
     {
diff --git a/renderdoc/driver/vulkan/vk_common.h b/renderdoc/driver/vulkan/vk_common.h
index ce528a66d..fee517e7b 100644
--- a/renderdoc/driver/vulkan/vk_common.h
+++ b/renderdoc/driver/vulkan/vk_common.h
@@ -430,7 +430,7 @@ struct DescriptorSetSlotImageInfo
 
 struct DescriptorSetSlot
 {
-  void RemoveBindRefs(VulkanResourceManager *rm, VkResourceRecord *record);
+  void RemoveBindRefs(std::set<ResourceId> &ids, VulkanResourceManager *rm, VkResourceRecord *record);
   void AddBindRefs(std::set<ResourceId> &ids, VulkanResourceManager *rm, VkResourceRecord *record,
                    FrameRefType ref);
 
diff --git a/renderdoc/driver/vulkan/vk_resources.h b/renderdoc/driver/vulkan/vk_resources.h
index 56c8dd4d1..b6a0de2e1 100644
--- a/renderdoc/driver/vulkan/vk_resources.h
+++ b/renderdoc/driver/vulkan/vk_resources.h
@@ -2201,7 +2201,7 @@ public:
     p.second = ComposeFrameRefsDisjoint(p.second, maxRef);
   }
 
-  void RemoveBindFrameRef(ResourceId id)
+  void RemoveBindFrameRef(std::set<ResourceId> &ids, ResourceId id)
   {
     // ignore any NULL IDs - probably an object that was
     // deleted since it was bound.
@@ -2220,7 +2220,10 @@ public:
     it->second.first--;
 
     if((it->second.first & ~DescriptorSetData::SPARSE_REF_BIT) == 0)
+    {
+      ids.insert(id);
       descInfo->bindFrameRefs.erase(it);
+    }
   }
 
   // we have a lot of 'cold' data in the resource record, as it can be accessed
diff --git a/renderdoc/driver/vulkan/wrappers/vk_descriptor_funcs.cpp b/renderdoc/driver/vulkan/wrappers/vk_descriptor_funcs.cpp
index ce26624ef..d67437ab3 100644
--- a/renderdoc/driver/vulkan/wrappers/vk_descriptor_funcs.cpp
+++ b/renderdoc/driver/vulkan/wrappers/vk_descriptor_funcs.cpp
@@ -1115,7 +1115,7 @@ void WrappedVulkan::vkUpdateDescriptorSets(VkDevice device, uint32_t writeCount,
 
         DescriptorSetSlot &bind = (*binding)[curIdx];
 
-        bind.RemoveBindRefs(GetResourceManager(), record);
+        bind.RemoveBindRefs(ids, GetResourceManager(), record);
 
         if(descWrite.descriptorType == VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER ||
            descWrite.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER)
@@ -1236,7 +1236,7 @@ void WrappedVulkan::vkUpdateDescriptorSets(VkDevice device, uint32_t writeCount,
 
         DescriptorSetSlot &bind = (*dstbinding)[curDstIdx];
 
-        bind.RemoveBindRefs(GetResourceManager(), dstrecord);
+        bind.RemoveBindRefs(ids, GetResourceManager(), dstrecord);
         bind = (*srcbinding)[curSrcIdx];
         bind.AddBindRefs(ids, GetResourceManager(), dstrecord, ref);
       }
@@ -1532,7 +1532,7 @@ void WrappedVulkan::vkUpdateDescriptorSetWithTemplate(
 
         DescriptorSetSlot &bind = (*binding)[curIdx];
 
-        bind.RemoveBindRefs(GetResourceManager(), record);
+        bind.RemoveBindRefs(ids, GetResourceManager(), record);
 
         if(entry.descriptorType == VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER ||
            entry.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER)