From 67246868f5e72a00bba643aec54bcc2255b33925 Mon Sep 17 00:00:00 2001 From: James Fulop Date: Tue, 27 Sep 2016 01:13:19 -0400 Subject: [PATCH] vs_out tri picking works from all angles --- renderdoc/data/glsl/mesh.comp | 3 --- renderdoc/data/hlsl/mesh.hlsl | 3 --- renderdoc/driver/d3d11/d3d11_analyse.cpp | 4 ++++ renderdoc/driver/gl/gl_debug.cpp | 6 ++++++ renderdoc/driver/vulkan/vk_debug.cpp | 5 ++++- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/renderdoc/data/glsl/mesh.comp b/renderdoc/data/glsl/mesh.comp index 52764bd5c..c23578d5e 100644 --- a/renderdoc/data/glsl/mesh.comp +++ b/renderdoc/data/glsl/mesh.comp @@ -84,9 +84,6 @@ void trianglePath(uint threadID) uint vertid0 = 0; uint vertid1 = 0; uint vertid2 = 0; - uint idx0 = 0; - uint idx1 = 0; - uint idx2 = 0; switch (meshpick.meshMode) { case MESH_TRIANGLE_LIST: diff --git a/renderdoc/data/hlsl/mesh.hlsl b/renderdoc/data/hlsl/mesh.hlsl index b41ee81ef..5172198ad 100644 --- a/renderdoc/data/hlsl/mesh.hlsl +++ b/renderdoc/data/hlsl/mesh.hlsl @@ -195,9 +195,6 @@ void trianglePath(uint threadID) uint vertid0 = 0; uint vertid1 = 0; uint vertid2 = 0; - uint idx0 = 0; - uint idx1 = 0; - uint idx2 = 0; switch (PickMeshMode) { case MESH_TRIANGLE_LIST: diff --git a/renderdoc/driver/d3d11/d3d11_analyse.cpp b/renderdoc/driver/d3d11/d3d11_analyse.cpp index 33f12cae0..1b293a4b2 100644 --- a/renderdoc/driver/d3d11/d3d11_analyse.cpp +++ b/renderdoc/driver/d3d11/d3d11_analyse.cpp @@ -2313,6 +2313,10 @@ uint32_t D3D11DebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg, Vec3f CameraToWorldFarPosition = InversePickMVP.Transform(Vec3f(pickXCanonical, pickYCanonical, 1), 1); RayDir = (CameraToWorldFarPosition - CameraToWorldNearPosition); + if(cfg.position.unproject && cfg.cam->GetForward().z < 0) + { + RayDir = -RayDir; + } RayDir.Normalise(); RayPos = CameraToWorldNearPosition; } diff --git a/renderdoc/driver/gl/gl_debug.cpp b/renderdoc/driver/gl/gl_debug.cpp index bade3f427..81e08f207 100644 --- a/renderdoc/driver/gl/gl_debug.cpp +++ b/renderdoc/driver/gl/gl_debug.cpp @@ -1061,9 +1061,15 @@ uint32_t GLReplay::PickVertex(uint32_t eventID, const MeshDisplay &cfg, uint32_t vec3 CameraToWorldNearPosition = InversePickMVP.Transform(Vec3f(pickXCanonical, pickYCanonical, -1), 1); + vec3 CameraToWorldFarPosition = InversePickMVP.Transform(Vec3f(pickXCanonical, pickYCanonical, 1), 1); + rayDir = (CameraToWorldFarPosition - CameraToWorldNearPosition); + if(cfg.position.unproject && cfg.cam->GetForward().z < 0) + { + rayDir = -rayDir; + } rayDir.Normalise(); rayPos = CameraToWorldNearPosition; } diff --git a/renderdoc/driver/vulkan/vk_debug.cpp b/renderdoc/driver/vulkan/vk_debug.cpp index bf6b4fe1a..112cdcb1a 100644 --- a/renderdoc/driver/vulkan/vk_debug.cpp +++ b/renderdoc/driver/vulkan/vk_debug.cpp @@ -3029,7 +3029,6 @@ FloatVector VulkanDebugManager::InterpretVertex(byte *data, uint32_t vert, const return ret; } -// TODO: VS_Out triangles doesn't pick correctly if you look back on the frustrum // TODO: Point meshes don't pick correctly uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg, uint32_t x, uint32_t y, uint32_t w, uint32_t h) @@ -3084,6 +3083,10 @@ uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg vec3 CameraToWorldFarPosition = InversePickMVP.Transform(Vec3f(pickXCanonical, pickYCanonical, 1), 1); rayDir = (CameraToWorldFarPosition - CameraToWorldNearPosition); + if(cfg.position.unproject && cfg.cam->GetForward().z < 0) + { + rayDir = -rayDir; + } rayDir.Normalise(); rayPos = CameraToWorldNearPosition; }