Don't generate potentially unsupported vertex formats to ignore W

* When using VS In we want to ignore the W component, but that already happens
  explicitly for vulkan/GL in the shader. Mirror that same solution to D3D
  instead of trying to force a 3-component format which may not be supported
  (e.g. on AMD R16G16B16_*)

qrenderdoc/Windows/BufferViewer.cpp

@@ -3230,7 +3230,6 @@ void BufferViewer::UI_CalculateMeshFormats()
         }
 
         m_VSInPosition.format = prop.format;
-        m_VSInPosition.format.compCount = qMin((uint8_t)3, m_VSInPosition.format.compCount);
       }
 
       elIdx = m_ModelVSIn->secondaryColumn();
@@ -3285,11 +3284,7 @@ void BufferViewer::UI_CalculateMeshFormats()
       // if geometry/tessellation is enabled, don't unproject VS output data
       if(m_Ctx.CurPipelineState().GetShader(ShaderStage::Tess_Eval) != ResourceId() ||
          m_Ctx.CurPipelineState().GetShader(ShaderStage::Geometry) != ResourceId())
-      {
         m_PostVSPosition.unproject = false;
-        // ignore any W component that might be there, let it be filled in with 1.0
-        m_PostVSPosition.format.compCount = qMin((uint8_t)3, m_VSInPosition.format.compCount);
-      }
 
       elIdx = m_ModelVSOut->secondaryColumn();
 

renderdoc/data/hlsl/hlsl_cbuffers.h

@@ -102,6 +102,8 @@ cbuffer MeshVertexCBuffer REG(b0)
   row_major float4x4 ModelViewProj;
 
   float2 SpriteSize;
+
+  uint homogenousInput;
 };
 
 cbuffer MeshGeometryCBuffer REG(b0)

renderdoc/data/hlsl/mesh.hlsl

@@ -44,7 +44,14 @@ meshV2F RENDERDOC_MeshVS(meshA2V IN, uint vid : SV_VertexID)
   float2 psprite[4] = {float2(-1.0f, -1.0f), float2(-1.0f, 1.0f), float2(1.0f, -1.0f),
                        float2(1.0f, 1.0f)};
 
-  OUT.pos = mul(IN.pos, ModelViewProj);
+  float4 pos = IN.pos;
+
+  if(homogenousInput == 0u)
+  {
+    pos = float4(pos.xyz, 1);
+  }
+
+  OUT.pos = mul(pos, ModelViewProj);
   OUT.pos.xy += SpriteSize.xy * 0.01f * psprite[vid % 4] * OUT.pos.w;
   OUT.norm = float3(0, 0, 1);
   OUT.secondary = IN.secondary;

renderdoc/driver/d3d11/d3d11_rendermesh.cpp

@@ -54,6 +54,7 @@ void D3D11Replay::RenderMesh(uint32_t eventId, const rdcarray<MeshFormat> &secon
 
   vertexData.ModelViewProj = projMat.Mul(camMat);
   vertexData.SpriteSize = Vec2f();
+  vertexData.homogenousInput = cfg.position.unproject;
 
   Vec4f col(0.0f, 0.0f, 0.0f, 1.0f);
   ID3D11Buffer *psCBuf = GetDebugManager()->MakeCBuffer(&col, sizeof(col));
@@ -320,6 +321,8 @@ void D3D11Replay::RenderMesh(uint32_t eventId, const rdcarray<MeshFormat> &secon
 
   m_pImmediateContext->RSSetState(m_MeshRender.WireframeRasterState);
 
+  vertexData.homogenousInput = 0U;
+
   // set up state for drawing helpers
   {
     vertexData.ModelViewProj = projMat.Mul(camMat);

renderdoc/driver/d3d12/d3d12_rendermesh.cpp

@@ -267,6 +267,7 @@ void D3D12Replay::RenderMesh(uint32_t eventId, const rdcarray<MeshFormat> &secon
 
   vertexData.ModelViewProj = projMat.Mul(camMat);
   vertexData.SpriteSize = Vec2f();
+  vertexData.homogenousInput = cfg.position.unproject;
 
   MeshPixelCBuffer pixelData;
 
@@ -543,6 +544,10 @@ void D3D12Replay::RenderMesh(uint32_t eventId, const rdcarray<MeshFormat> &secon
 
   pixelData.MeshDisplayFormat = MESHDISPLAY_SOLID;
 
+  vertexData.homogenousInput = 0U;
+
+  vsCB = GetDebugManager()->UploadConstants(&vertexData, sizeof(vertexData));
+
   // cache pipelines for use in drawing wireframe helpers
   cache = GetDebugManager()->CacheMeshDisplayPipelines(helper, helper);
 
@@ -584,6 +589,9 @@ void D3D12Replay::RenderMesh(uint32_t eventId, const rdcarray<MeshFormat> &secon
 
     list->SetPipelineState(cache.pipes[MeshDisplayPipelines::ePipe_WireDepth]);
 
+    list->SetGraphicsRootConstantBufferView(0, vsCB);
+    list->SetGraphicsRootConstantBufferView(1, vsCB);
+
     list->DrawInstanced(24, 1, 0, 0);
   }
 
@@ -606,6 +614,9 @@ void D3D12Replay::RenderMesh(uint32_t eventId, const rdcarray<MeshFormat> &secon
 
     list->SetPipelineState(cache.pipes[MeshDisplayPipelines::ePipe_Wire]);
 
+    list->SetGraphicsRootConstantBufferView(0, vsCB);
+    list->SetGraphicsRootConstantBufferView(1, vsCB);
+
     pixelData.MeshColour = Vec3f(1.0f, 0.0f, 0.0f);
     list->SetGraphicsRoot32BitConstants(2, 4, &pixelData, 0);
     list->DrawInstanced(2, 1, 0, 0);
@@ -655,6 +666,9 @@ void D3D12Replay::RenderMesh(uint32_t eventId, const rdcarray<MeshFormat> &secon
 
     list->SetPipelineState(cache.pipes[MeshDisplayPipelines::ePipe_Wire]);
 
+    list->SetGraphicsRootConstantBufferView(0, vsCB);
+    list->SetGraphicsRootConstantBufferView(1, vsCB);
+
     list->DrawInstanced(24, 1, 0, 0);
   }