Emulate bad vertex attribute casts on GL in mesh viewer

* On GL you can specify a vertex attribute that's stored as int but gets
  converted to float with glVertexAttribFormat instead of glVertexAttribIFormat.
  However if the shader accepts an int this is invalid and the value is
  undefined - we emulate this as the float bits being read as int directly, but
  that's not guaranteed behaviour.
* Normally we don't emulate this kind of mis-cast behaviour and just display the
  type of data passed to the shader, but in this case GL lets you specify three
  types (stored as int, cast to float, read as int) so our normal behaviour of
  just showing the input can be more misleading than normal.

qrenderdoc/Windows/BufferViewer.cpp

@@ -455,6 +455,7 @@ struct BufferElementProperties
   int buffer = 0;
   ShaderBuiltin systemValue = ShaderBuiltin::Undefined;
   bool perinstance = false;
+  bool floatCastWrong = false;
   int instancerate = 1;
 };
 
@@ -733,7 +734,14 @@ static QString interpretVariant(const QVariant &v, const ShaderConstant &el,
   }
   else if(vt == QMetaType::UInt || vt == QMetaType::UShort || vt == QMetaType::UChar)
   {
-    uint u = v.toUInt();
+    uint32_t u = v.toUInt();
+
+    if(prop.floatCastWrong)
+    {
+      float f = (float)u;
+      memcpy(&u, &f, sizeof(f));
+    }
+
     if(el.type.descriptor.displayAsHex && prop.format.type == ResourceFormatType::Regular)
       ret = Formatter::HexFormat(u, prop.format.compByteWidth);
     else
@@ -741,7 +749,14 @@ static QString interpretVariant(const QVariant &v, const ShaderConstant &el,
   }
   else if(vt == QMetaType::Int || vt == QMetaType::Short || vt == QMetaType::SChar)
   {
-    int i = v.toInt();
+    int32_t i = v.toInt();
+
+    if(prop.floatCastWrong)
+    {
+      float f = (float)i;
+      memcpy(&i, &f, sizeof(f));
+    }
+
     if(i >= 0)
       ret = lit(" ") + Formatter::Format(i);
     else
@@ -1533,6 +1548,7 @@ static void ConfigureMeshColumns(ICaptureContext &ctx, PopulateBufferData *bufda
     p.buffer = a.vertexBuffer;
     p.perinstance = a.perInstance;
     p.instancerate = a.instanceRate;
+    p.floatCastWrong = a.floatCastWrong;
     p.format = a.format;
 
     bufdata->vsinConfig.genericsEnabled[bufdata->vsinConfig.columns.count()] = false;

renderdoc/api/replay/common_pipestate.h

@@ -482,6 +482,16 @@ from the vertex buffer before advancing to the next value.
   PixelValue genericValue;
   DOCUMENT("``True`` if this attribute is using :data:`genericValue` for its data.");
   bool genericEnabled = false;
+  DOCUMENT(R"(Only valid for attributes on OpenGL. If the attribute has been set up for integers to
+be converted to floats (glVertexAttribFormat with GL_INT) we store the format as integers. This is
+fine if the application has a float input in the shader it just means we display the raw integer
+instead of the casted float. However if the shader has an integer input this is invalid and it will
+read something undefined - possibly the int bits of the casted float.
+
+This property is set to ``True`` if the cast happens to an integer input and that bad cast needs to
+be emulated.
+)");
+  bool floatCastWrong = false;
   DOCUMENT("``True`` if this attribute is enabled and used by the vertex shader.");
   bool used;
 };

renderdoc/api/replay/pipestate.inl

@@ -800,16 +800,23 @@ rdcarray<VertexInputAttribute> PipeState::GetVertexInputs() const
         {
           int attrib = m_GL->vertexShader.bindpointMapping.inputAttributes[i];
 
+          const SigParameter &sigParam = m_GL->vertexShader.reflection->inputSignature[attrib];
+
           if(attrib >= 0 && attrib < m_GL->vertexShader.reflection->inputSignature.count())
-            ret[a].name = m_GL->vertexShader.reflection->inputSignature[attrib].varName;
+            ret[a].name = sigParam.varName;
 
           if(attrib == -1)
             continue;
 
+          VarType varType = sigParam.varType;
+
+          if(attrs[i].floatCast && (VarTypeCompType(sigParam.varType) == CompType::UInt ||
+                                    VarTypeCompType(sigParam.varType) == CompType::SInt))
+            ret[a].floatCastWrong = true;
+
           if(!attrs[i].enabled)
           {
-            uint32_t compCount = m_GL->vertexShader.reflection->inputSignature[attrib].compCount;
-            VarType varType = m_GL->vertexShader.reflection->inputSignature[attrib].varType;
+            uint32_t compCount = sigParam.compCount;
 
             for(uint32_t c = 0; c < compCount; c++)
             {