From 0804ef63be1d579141a63e3f27d55d6655524c42 Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Mon, 16 Aug 2021 10:54:47 +0100
Subject: [PATCH] Emulate bad vertex attribute casts on GL in mesh viewer

* On GL you can specify a vertex attribute that's stored as int but gets
  converted to float with glVertexAttribFormat instead of glVertexAttribIFormat.
  However if the shader accepts an int this is invalid and the value is
  undefined - we emulate this as the float bits being read as int directly, but
  that's not guaranteed behaviour.
* Normally we don't emulate this kind of mis-cast behaviour and just display the
  type of data passed to the shader, but in this case GL lets you specify three
  types (stored as int, cast to float, read as int) so our normal behaviour of
  just showing the input can be more misleading than normal.
---
 qrenderdoc/Windows/BufferViewer.cpp     | 20 ++++++++++++++++++--
 renderdoc/api/replay/common_pipestate.h | 10 ++++++++++
 renderdoc/api/replay/pipestate.inl      | 13 ++++++++++---
 3 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/qrenderdoc/Windows/BufferViewer.cpp b/qrenderdoc/Windows/BufferViewer.cpp
index 9caf54bb1..cd4905513 100644
--- a/qrenderdoc/Windows/BufferViewer.cpp
+++ b/qrenderdoc/Windows/BufferViewer.cpp
@@ -455,6 +455,7 @@ struct BufferElementProperties
   int buffer = 0;
   ShaderBuiltin systemValue = ShaderBuiltin::Undefined;
   bool perinstance = false;
+  bool floatCastWrong = false;
   int instancerate = 1;
 };
 
@@ -733,7 +734,14 @@ static QString interpretVariant(const QVariant &v, const ShaderConstant &el,
   }
   else if(vt == QMetaType::UInt || vt == QMetaType::UShort || vt == QMetaType::UChar)
   {
-    uint u = v.toUInt();
+    uint32_t u = v.toUInt();
+
+    if(prop.floatCastWrong)
+    {
+      float f = (float)u;
+      memcpy(&u, &f, sizeof(f));
+    }
+
     if(el.type.descriptor.displayAsHex && prop.format.type == ResourceFormatType::Regular)
       ret = Formatter::HexFormat(u, prop.format.compByteWidth);
     else
@@ -741,7 +749,14 @@ static QString interpretVariant(const QVariant &v, const ShaderConstant &el,
   }
   else if(vt == QMetaType::Int || vt == QMetaType::Short || vt == QMetaType::SChar)
   {
-    int i = v.toInt();
+    int32_t i = v.toInt();
+
+    if(prop.floatCastWrong)
+    {
+      float f = (float)i;
+      memcpy(&i, &f, sizeof(f));
+    }
+
     if(i >= 0)
       ret = lit(" ") + Formatter::Format(i);
     else
@@ -1533,6 +1548,7 @@ static void ConfigureMeshColumns(ICaptureContext &ctx, PopulateBufferData *bufda
     p.buffer = a.vertexBuffer;
     p.perinstance = a.perInstance;
     p.instancerate = a.instanceRate;
+    p.floatCastWrong = a.floatCastWrong;
     p.format = a.format;
 
     bufdata->vsinConfig.genericsEnabled[bufdata->vsinConfig.columns.count()] = false;
diff --git a/renderdoc/api/replay/common_pipestate.h b/renderdoc/api/replay/common_pipestate.h
index 1cb58d347..7ab1271f0 100644
--- a/renderdoc/api/replay/common_pipestate.h
+++ b/renderdoc/api/replay/common_pipestate.h
@@ -482,6 +482,16 @@ from the vertex buffer before advancing to the next value.
   PixelValue genericValue;
   DOCUMENT("``True`` if this attribute is using :data:`genericValue` for its data.");
   bool genericEnabled = false;
+  DOCUMENT(R"(Only valid for attributes on OpenGL. If the attribute has been set up for integers to
+be converted to floats (glVertexAttribFormat with GL_INT) we store the format as integers. This is
+fine if the application has a float input in the shader it just means we display the raw integer
+instead of the casted float. However if the shader has an integer input this is invalid and it will
+read something undefined - possibly the int bits of the casted float.
+
+This property is set to ``True`` if the cast happens to an integer input and that bad cast needs to
+be emulated.
+)");
+  bool floatCastWrong = false;
   DOCUMENT("``True`` if this attribute is enabled and used by the vertex shader.");
   bool used;
 };
diff --git a/renderdoc/api/replay/pipestate.inl b/renderdoc/api/replay/pipestate.inl
index 5d58e321d..8d145a3de 100644
--- a/renderdoc/api/replay/pipestate.inl
+++ b/renderdoc/api/replay/pipestate.inl
@@ -800,16 +800,23 @@ rdcarray<VertexInputAttribute> PipeState::GetVertexInputs() const
         {
           int attrib = m_GL->vertexShader.bindpointMapping.inputAttributes[i];
 
+          const SigParameter &sigParam = m_GL->vertexShader.reflection->inputSignature[attrib];
+
           if(attrib >= 0 && attrib < m_GL->vertexShader.reflection->inputSignature.count())
-            ret[a].name = m_GL->vertexShader.reflection->inputSignature[attrib].varName;
+            ret[a].name = sigParam.varName;
 
           if(attrib == -1)
             continue;
 
+          VarType varType = sigParam.varType;
+
+          if(attrs[i].floatCast && (VarTypeCompType(sigParam.varType) == CompType::UInt ||
+                                    VarTypeCompType(sigParam.varType) == CompType::SInt))
+            ret[a].floatCastWrong = true;
+
           if(!attrs[i].enabled)
           {
-            uint32_t compCount = m_GL->vertexShader.reflection->inputSignature[attrib].compCount;
-            VarType varType = m_GL->vertexShader.reflection->inputSignature[attrib].varType;
+            uint32_t compCount = sigParam.compCount;
 
             for(uint32_t c = 0; c < compCount; c++)
             {