lee/renderdoc
1
0
Fork 0
mirror of https://github.com/baldurk/renderdoc.git synced 2026-08-23 15:06:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

vk handles unproject sign more robustly

Browse Source 
vk format

vk handles unproject sign more robustly
This commit is contained in:
James Fulop
2016-09-30 10:58:29 +02:00
committed by baldurk
parent 8054365bd6
commit ceeba57c12
1 changed files with 38 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
renderdoc/driver/vulkan/vk_debug.cpp
+38 -13
View File
@@ -3039,7 +3039,7 @@ uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg
Matrix4f projMat = Matrix4f::Perspective(90.0f, 0.1f, 100000.0f, float(w) / float(h));
Matrix4f camMat = cfg.cam ? cfg.cam->GetMatrix() : Matrix4f::Identity();
Matrix4f PickMVP = projMat.Mul(camMat);
Matrix4f pickMVP = projMat.Mul(camMat);
ResourceFormat resFmt;
resFmt.compByteWidth = cfg.position.compByteWidth;
@@ -3052,6 +3052,7 @@ uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg
resFmt.specialFormat = cfg.position.specialFormat;
}
Matrix4f pickMVPProj;
if(cfg.position.unproject)
{
// the derivation of the projection matrix might not be right (hell, it could be an
@@ -3062,14 +3063,14 @@ uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg
if(cfg.ortho)
guessProj = Matrix4f::Orthographic(cfg.position.nearPlane, cfg.position.farPlane);
PickMVP = projMat.Mul(camMat.Mul(guessProj.Inverse()));
pickMVPProj = projMat.Mul(camMat.Mul(guessProj.Inverse()));
}
vec3 rayPos;
vec3 rayDir;
// convert mouse pos to world space ray
{
Matrix4f InversePickMVP = PickMVP.Inverse();
Matrix4f inversePickMVP = pickMVP.Inverse();
float pickX = ((float)x) / ((float)w);
float pickXCanonical = RDCLERP(-1.0f, 1.0f, pickX);
@@ -3078,17 +3079,41 @@ uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg
// flip the Y axis
float pickYCanonical = RDCLERP(1.0f, -1.0f, pickY);
vec3 CameraToWorldNearPosition =
InversePickMVP.Transform(Vec3f(pickXCanonical, pickYCanonical, -1), 1);
vec3 CameraToWorldFarPosition =
InversePickMVP.Transform(Vec3f(pickXCanonical, pickYCanonical, 1), 1);
rayDir = (CameraToWorldFarPosition - CameraToWorldNearPosition);
if(cfg.position.unproject && cfg.cam->GetForward().z < 0)
vec3 cameraToWorldNearPosition =
inversePickMVP.Transform(Vec3f(pickXCanonical, pickYCanonical, -1), 1);
vec3 cameraToWorldFarPosition =
inversePickMVP.Transform(Vec3f(pickXCanonical, pickYCanonical, 1), 1);
vec3 testDir = (cameraToWorldFarPosition - cameraToWorldNearPosition);
testDir.Normalise();
/* Calculate the ray direction first in the regular way (above), so we can use the
the output for testing if the ray we are picking is negative or not. This is similar
to checking against the forward direction of the camera, but more robust
*/
if(cfg.position.unproject)
{
rayDir = -rayDir;
Matrix4f inversePickMVPGuess = pickMVPProj.Inverse();
vec3 nearPosProj = inversePickMVPGuess.Transform(Vec3f(pickXCanonical, pickYCanonical, -1), 1);
vec3 farPosProj = inversePickMVPGuess.Transform(Vec3f(pickXCanonical, pickYCanonical, 1), 1);
rayDir = (farPosProj - nearPosProj);
rayDir.Normalise();
if(testDir.z < 0)
{
rayDir = -rayDir;
}
rayPos = nearPosProj;
}
else
{
rayDir = testDir;
rayPos = cameraToWorldNearPosition;
}
rayDir.Normalise();
rayPos = CameraToWorldNearPosition;
}
MeshPickUBOData *ubo = (MeshPickUBOData *)m_MeshPickUBO.Map();
@@ -3135,7 +3160,7 @@ uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg
// line/point data
ubo->unproject = cfg.position.unproject;
ubo->mvp = PickMVP;
ubo->mvp = cfg.position.unproject ? pickMVPProj : pickMVP;
ubo->coords = Vec2f((float)x, (float)y);
ubo->viewport = Vec2f((float)w, (float)h);