From e78a48bfc31c8e7f9cd6727591b8c2825bfa8f97 Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Fri, 23 Apr 2021 16:59:57 +0100
Subject: [PATCH] Implement proper out-of-bounds value filling for vulkan
 post-vs fetch

---
 renderdoc/driver/vulkan/vk_postvs.cpp | 38 ++++++++++++++++++++++-----
 renderdoc/maths/formatpacking.cpp     |  3 +++
 renderdoc/replay/replay_driver.cpp    |  8 ++++++
 3 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/renderdoc/driver/vulkan/vk_postvs.cpp b/renderdoc/driver/vulkan/vk_postvs.cpp
index cd416fd7b..25baa4a16 100644
--- a/renderdoc/driver/vulkan/vk_postvs.cpp
+++ b/renderdoc/driver/vulkan/vk_postvs.cpp
@@ -2079,30 +2079,49 @@ void VulkanReplay::FetchVSOut(uint32_t eventId, VulkanRenderState &state)
         vkr = m_pDriver->vkBindBufferMemory(dev, vbuffers[attr].buf, vbuffers[attr].mem, 0);
         RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
-        byte *compactedData = NULL;
-        vkr = m_pDriver->vkMapMemory(m_Device, vbuffers[attr].mem, 0, VK_WHOLE_SIZE, 0,
-                                     (void **)&compactedData);
+        byte *dst = NULL;
+        vkr =
+            m_pDriver->vkMapMemory(m_Device, vbuffers[attr].mem, 0, VK_WHOLE_SIZE, 0, (void **)&dst);
         RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
-        if(compactedData && origVBEnd)
+        const byte *dstBase = dst;
+        (void)dstBase;
+
+        const byte *dstEnd = dst + bufInfo.size;
+
+        if(dst)
         {
+          FloatVector defaultValue(0.0f, 0.0f, 0.0f, 1.0f);
+          if(fmt.compType == CompType::UInt || fmt.compType == CompType::SInt || fmt.compCount == 4)
+            defaultValue.w = 0.0f;
+
           const byte *src = origVBBegin;
-          byte *dst = compactedData;
-          const byte *dstEnd = dst + bufInfo.size;
 
           // fast memcpy compaction case for regular 32-bit types. Any type like R32G32B32 or so on
           // can be memcpy'd into place and read, since we discard any unused components and there's
           // no re-interpretation needed.
           if(fmt.type == ResourceFormatType::Regular && fmt.compByteWidth == 4)
           {
+            size_t expandedComponentBytes = sizeof(FloatVector) - origElemSize;
+
             while(src < origVBEnd && dst < dstEnd)
             {
+              if(expandedComponentBytes > 0)
+                memcpy(dst + origElemSize, ((byte *)&defaultValue) + origElemSize,
+                       expandedComponentBytes);
               memcpy(dst, src, origElemSize);
 
               // advance by the *destination* element size of 16 bytes
               dst += elemSize;
               src += stride;
             }
+
+            // fill the rest with default values
+            while(dst < dstEnd)
+            {
+              memcpy(dst, &defaultValue, sizeof(FloatVector));
+              dst += elemSize;
+            }
           }
           else
           {
@@ -2203,6 +2222,13 @@ void VulkanReplay::FetchVSOut(uint32_t eventId, VulkanRenderState &state)
                 dst += sizeof(FloatVector);
                 src += stride;
               }
+
+              // fill the rest with default values
+              while(dst < dstEnd)
+              {
+                memcpy(dst, &defaultValue, sizeof(FloatVector));
+                dst += elemSize;
+              }
             }
           }
         }
diff --git a/renderdoc/maths/formatpacking.cpp b/renderdoc/maths/formatpacking.cpp
index d8ac6ee28..e901ce2f0 100644
--- a/renderdoc/maths/formatpacking.cpp
+++ b/renderdoc/maths/formatpacking.cpp
@@ -308,6 +308,9 @@ FloatVector DecodeFormattedComponents(const ResourceFormat &fmt, const byte *dat
 {
   FloatVector ret(0.0f, 0.0f, 0.0f, 1.0f);
 
+  if(fmt.compType == CompType::UInt || fmt.compType == CompType::SInt || fmt.compCount == 4)
+    ret.w = 0.0f;
+
   const uint64_t dummy = 0;
   if(!data)
     data = (const byte *)&dummy;
diff --git a/renderdoc/replay/replay_driver.cpp b/renderdoc/replay/replay_driver.cpp
index c7026676c..4f31f3080 100644
--- a/renderdoc/replay/replay_driver.cpp
+++ b/renderdoc/replay/replay_driver.cpp
@@ -587,6 +587,10 @@ FloatVector HighlightCache::InterpretVertex(const byte *data, uint32_t vert, con
 {
   FloatVector ret(0.0f, 0.0f, 0.0f, 1.0f);
 
+  if(cfg.position.format.compType == CompType::UInt ||
+     cfg.position.format.compType == CompType::SInt || cfg.position.format.compCount == 4)
+    ret.w = 0.0f;
+
   if(useidx && idxData)
   {
     if(vert >= (uint32_t)indices.size())
@@ -623,6 +627,10 @@ FloatVector HighlightCache::InterpretVertex(const byte *data, uint32_t vert,
   if(data + fmt.ElementSize() > end)
   {
     valid = false;
+
+    if(fmt.compType == CompType::UInt || fmt.compType == CompType::SInt || fmt.compCount == 4)
+      return FloatVector(0.0f, 0.0f, 0.0f, 0.0f);
+
     return FloatVector(0.0f, 0.0f, 0.0f, 1.0f);
   }