From a41c35b76469878d634b49ac325bdaa2fa4c0c67 Mon Sep 17 00:00:00 2001 From: Steve Legg Date: Fri, 20 Jan 2017 02:36:55 +0000 Subject: [PATCH] Fix mesh picking with perspective camera --- renderdoc/driver/d3d11/d3d11_analyse.cpp | 4 +++- renderdoc/driver/d3d12/d3d12_debug.cpp | 4 +++- renderdoc/driver/gl/gl_debug.cpp | 4 +++- renderdoc/driver/vulkan/vk_debug.cpp | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/renderdoc/driver/d3d11/d3d11_analyse.cpp b/renderdoc/driver/d3d11/d3d11_analyse.cpp index c669e251b..9d4d26251 100644 --- a/renderdoc/driver/d3d11/d3d11_analyse.cpp +++ b/renderdoc/driver/d3d11/d3d11_analyse.cpp @@ -2300,7 +2300,9 @@ uint32_t D3D11DebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg, // the derivation of the projection matrix might not be right (hell, it could be an // orthographic projection). But it'll be close enough likely. Matrix4f guessProj = - Matrix4f::Perspective(cfg.fov, cfg.position.nearPlane, cfg.position.farPlane, cfg.aspect); + cfg.position.farPlane != FLT_MAX + ? Matrix4f::Perspective(cfg.fov, cfg.position.nearPlane, cfg.position.farPlane, cfg.aspect) + : Matrix4f::ReversePerspective(cfg.fov, cfg.position.nearPlane, cfg.aspect); if(cfg.ortho) guessProj = Matrix4f::Orthographic(cfg.position.nearPlane, cfg.position.farPlane); diff --git a/renderdoc/driver/d3d12/d3d12_debug.cpp b/renderdoc/driver/d3d12/d3d12_debug.cpp index 4728aff22..25bc48ec8 100644 --- a/renderdoc/driver/d3d12/d3d12_debug.cpp +++ b/renderdoc/driver/d3d12/d3d12_debug.cpp @@ -2435,7 +2435,9 @@ uint32_t D3D12DebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg, // the derivation of the projection matrix might not be right (hell, it could be an // orthographic projection). But it'll be close enough likely. Matrix4f guessProj = - Matrix4f::Perspective(cfg.fov, cfg.position.nearPlane, cfg.position.farPlane, cfg.aspect); + cfg.position.farPlane != FLT_MAX + ? Matrix4f::Perspective(cfg.fov, cfg.position.nearPlane, cfg.position.farPlane, cfg.aspect) + : Matrix4f::ReversePerspective(cfg.fov, cfg.position.nearPlane, cfg.aspect); if(cfg.ortho) guessProj = Matrix4f::Orthographic(cfg.position.nearPlane, cfg.position.farPlane); diff --git a/renderdoc/driver/gl/gl_debug.cpp b/renderdoc/driver/gl/gl_debug.cpp index 057c98606..8d4ebb4c5 100644 --- a/renderdoc/driver/gl/gl_debug.cpp +++ b/renderdoc/driver/gl/gl_debug.cpp @@ -1149,7 +1149,9 @@ uint32_t GLReplay::PickVertex(uint32_t eventID, const MeshDisplay &cfg, uint32_t // the derivation of the projection matrix might not be right (hell, it could be an // orthographic projection). But it'll be close enough likely. Matrix4f guessProj = - Matrix4f::Perspective(cfg.fov, cfg.position.nearPlane, cfg.position.farPlane, cfg.aspect); + cfg.position.farPlane != FLT_MAX + ? Matrix4f::Perspective(cfg.fov, cfg.position.nearPlane, cfg.position.farPlane, cfg.aspect) + : Matrix4f::ReversePerspective(cfg.fov, cfg.position.nearPlane, cfg.aspect); if(cfg.ortho) guessProj = Matrix4f::Orthographic(cfg.position.nearPlane, cfg.position.farPlane); diff --git a/renderdoc/driver/vulkan/vk_debug.cpp b/renderdoc/driver/vulkan/vk_debug.cpp index e0a63452b..ee56db8db 100644 --- a/renderdoc/driver/vulkan/vk_debug.cpp +++ b/renderdoc/driver/vulkan/vk_debug.cpp @@ -3974,7 +3974,9 @@ uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg // the derivation of the projection matrix might not be right (hell, it could be an // orthographic projection). But it'll be close enough likely. Matrix4f guessProj = - Matrix4f::Perspective(cfg.fov, cfg.position.nearPlane, cfg.position.farPlane, cfg.aspect); + cfg.position.farPlane != FLT_MAX + ? Matrix4f::Perspective(cfg.fov, cfg.position.nearPlane, cfg.position.farPlane, cfg.aspect) + : Matrix4f::ReversePerspective(cfg.fov, cfg.position.nearPlane, cfg.aspect); if(cfg.ortho) guessProj = Matrix4f::Orthographic(cfg.position.nearPlane, cfg.position.farPlane);