From ceeba57c129ea053538885fa8109b23b7d80948d Mon Sep 17 00:00:00 2001
From: James Fulop <jameslynnfulop@gmail.com>
Date: Thu, 29 Sep 2016 15:12:17 -0400
Subject: [PATCH] vk handles unproject sign more robustly

vk format

vk handles unproject sign more robustly
---
 renderdoc/driver/vulkan/vk_debug.cpp | 51 +++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 13 deletions(-)

diff --git a/renderdoc/driver/vulkan/vk_debug.cpp b/renderdoc/driver/vulkan/vk_debug.cpp
index 112cdcb1a..cd37ea986 100644
--- a/renderdoc/driver/vulkan/vk_debug.cpp
+++ b/renderdoc/driver/vulkan/vk_debug.cpp
@@ -3039,7 +3039,7 @@ uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg
   Matrix4f projMat = Matrix4f::Perspective(90.0f, 0.1f, 100000.0f, float(w) / float(h));
 
   Matrix4f camMat = cfg.cam ? cfg.cam->GetMatrix() : Matrix4f::Identity();
-  Matrix4f PickMVP = projMat.Mul(camMat);
+  Matrix4f pickMVP = projMat.Mul(camMat);
 
   ResourceFormat resFmt;
   resFmt.compByteWidth = cfg.position.compByteWidth;
@@ -3052,6 +3052,7 @@ uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg
     resFmt.specialFormat = cfg.position.specialFormat;
   }
 
+  Matrix4f pickMVPProj;
   if(cfg.position.unproject)
   {
     // the derivation of the projection matrix might not be right (hell, it could be an
@@ -3062,14 +3063,14 @@ uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg
     if(cfg.ortho)
       guessProj = Matrix4f::Orthographic(cfg.position.nearPlane, cfg.position.farPlane);
 
-    PickMVP = projMat.Mul(camMat.Mul(guessProj.Inverse()));
+    pickMVPProj = projMat.Mul(camMat.Mul(guessProj.Inverse()));
   }
 
   vec3 rayPos;
   vec3 rayDir;
   // convert mouse pos to world space ray
   {
-    Matrix4f InversePickMVP = PickMVP.Inverse();
+    Matrix4f inversePickMVP = pickMVP.Inverse();
 
     float pickX = ((float)x) / ((float)w);
     float pickXCanonical = RDCLERP(-1.0f, 1.0f, pickX);
@@ -3078,17 +3079,41 @@ uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg
     // flip the Y axis
     float pickYCanonical = RDCLERP(1.0f, -1.0f, pickY);
 
-    vec3 CameraToWorldNearPosition =
-        InversePickMVP.Transform(Vec3f(pickXCanonical, pickYCanonical, -1), 1);
-    vec3 CameraToWorldFarPosition =
-        InversePickMVP.Transform(Vec3f(pickXCanonical, pickYCanonical, 1), 1);
-    rayDir = (CameraToWorldFarPosition - CameraToWorldNearPosition);
-    if(cfg.position.unproject && cfg.cam->GetForward().z < 0)
+    vec3 cameraToWorldNearPosition =
+        inversePickMVP.Transform(Vec3f(pickXCanonical, pickYCanonical, -1), 1);
+
+    vec3 cameraToWorldFarPosition =
+        inversePickMVP.Transform(Vec3f(pickXCanonical, pickYCanonical, 1), 1);
+
+    vec3 testDir = (cameraToWorldFarPosition - cameraToWorldNearPosition);
+    testDir.Normalise();
+
+    /* Calculate the ray direction first in the regular way (above), so we can use the
+    the output for testing if the ray we are picking is negative or not. This is similar
+    to checking against the forward direction of the camera, but more robust
+    */
+    if(cfg.position.unproject)
     {
-      rayDir = -rayDir;
+      Matrix4f inversePickMVPGuess = pickMVPProj.Inverse();
+
+      vec3 nearPosProj = inversePickMVPGuess.Transform(Vec3f(pickXCanonical, pickYCanonical, -1), 1);
+
+      vec3 farPosProj = inversePickMVPGuess.Transform(Vec3f(pickXCanonical, pickYCanonical, 1), 1);
+
+      rayDir = (farPosProj - nearPosProj);
+      rayDir.Normalise();
+
+      if(testDir.z < 0)
+      {
+        rayDir = -rayDir;
+      }
+      rayPos = nearPosProj;
+    }
+    else
+    {
+      rayDir = testDir;
+      rayPos = cameraToWorldNearPosition;
     }
-    rayDir.Normalise();
-    rayPos = CameraToWorldNearPosition;
   }
 
   MeshPickUBOData *ubo = (MeshPickUBOData *)m_MeshPickUBO.Map();
@@ -3135,7 +3160,7 @@ uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg
 
   // line/point data
   ubo->unproject = cfg.position.unproject;
-  ubo->mvp = PickMVP;
+  ubo->mvp = cfg.position.unproject ? pickMVPProj : pickMVP;
   ubo->coords = Vec2f((float)x, (float)y);
   ubo->viewport = Vec2f((float)w, (float)h);