Handle vertex picking in world space for post-projection data

* Previously the vertex picking was always being done in NDC space, but now for
  perspective projections we unproject into world space and raycast there for
  more reliable results with unsual projections.

renderdoc/data/glsl/glsl_ubos.h

@@ -113,9 +113,10 @@ BINDING(0) uniform MeshPickUBOData
 
   uint meshMode;    // triangles, triangle strip, fan, etc...
   uint unproject;
-  vec2 padding;
+  uint flipY;
+  uint ortho;
 
-  mat4 mvp;
+  mat4 transformMat;
 }
 INST_NAME(meshpick);
 

renderdoc/data/glsl/mesh.comp

@@ -87,7 +87,7 @@ bool TriangleRayIntersect(vec3 A, vec3 B, vec3 C, vec3 RayPosition, vec3 RayDire
     if(u >= 0.0f && u <= 1.0f && v >= 0.0f && u + v <= 1.0f)
     {
       float t = dot(v0v2, qvec) * invDet;
-      if(t > 0.0f)
+      if(t >= 0.0f)
       {
         HitPosition = RayPosition + (RayDirection * t);
         Result = true;
@@ -164,6 +164,13 @@ void trianglePath(uint threadID)
   bool hit;
   if(meshpick.unproject == 1u)
   {
+    if(meshpick.ortho == 0u)
+    {
+      pos0 = meshpick.transformMat * pos0;
+      pos1 = meshpick.transformMat * pos1;
+      pos2 = meshpick.transformMat * pos2;
+    }
+
     pos0.xyz /= pos0.w;
     pos1.xyz /= pos1.w;
     pos2.xyz /= pos2.w;
@@ -213,12 +220,13 @@ void defaultPath(uint threadID)
     pos = vec4(pos.x, -pos.y, pos.z, pos.w);
 #endif
 
-  vec4 wpos = meshpick.mvp * pos;
+  vec4 wpos = meshpick.transformMat * pos;
 
   if(meshpick.unproject == 1u)
     wpos.xyz /= wpos.www;
 
-  wpos.xy *= vec2(1.0f, -1.0f);
+  if(meshpick.flipY == 0u)
+    wpos.xy *= vec2(1.0f, -1.0f);
 
   vec2 scr = (wpos.xy + 1.0f) * 0.5f * meshpick.viewport;
 

renderdoc/data/hlsl/hlsl_cbuffers.h

@@ -130,9 +130,10 @@ cbuffer MeshPickData REG(b0)
 
   uint PickMeshMode;
   uint PickUnproject;
-  float2 Padding;
+  uint PickFlipY;
+  uint PickOrtho;
 
-  row_major float4x4 PickMVP;
+  row_major float4x4 PickTransformMat;
 };
 
 #define HEATMAP_DISABLED 0

renderdoc/data/hlsl/mesh.hlsl

@@ -173,7 +173,7 @@ bool TriangleRayIntersect(float3 A, float3 B, float3 C, float3 RayPosition, floa
     if(u >= 0 && u <= 1 && v >= 0 && u + v <= 1)
     {
       float t = dot(v0v2, qvec) * invDet;
-      if(t > 0)
+      if(t >= 0)
       {
         HitPosition = RayPosition + (RayDirection * t);
         Result = true;
@@ -234,6 +234,13 @@ void trianglePath(uint threadID)
   bool hit;
   if(PickUnproject == 1)
   {
+    if(PickOrtho == 0)
+    {
+      pos0 = mul(pos0, PickTransformMat);
+      pos1 = mul(pos1, PickTransformMat);
+      pos2 = mul(pos2, PickTransformMat);
+    }
+
     pos0.xyz /= pos0.w;
     pos1.xyz /= pos1.w;
     pos2.xyz /= pos2.w;
@@ -275,12 +282,13 @@ void defaultPath(uint threadID)
 
   float4 pos = vertex[idx];
 
-  float4 wpos = mul(pos, PickMVP);
+  float4 wpos = mul(pos, PickTransformMat);
 
   if(PickUnproject == 1)
     wpos.xyz /= wpos.www;
 
-  wpos.xy *= float2(1.0f, -1.0f);
+  if(PickFlipY == 0)
+    wpos.xy *= float2(1.0f, -1.0f);
 
   float2 scr = (wpos.xy + 1.0f) * 0.5f * PickViewport;