From ef185bca348aa7957dcc67a1385fa64f4b9569af Mon Sep 17 00:00:00 2001 From: James Fulop Date: Mon, 12 Sep 2016 21:22:02 -0400 Subject: [PATCH] improved vertex picking, only Opengl VS Input The new vertex picking solution raycasts from the mouse's position, picking vertices based on 1. The triangle it has intersected (take the nearest triangle) 2. The nearest vertex to the intersection point on the triangle This is replacing a solution that was based on how close the vertex was to the mouse in screen space and its depth. This commit only supports the OpenGL path in the VS Input window. I have disabled vertex picking in Vulkan for now, as there are some cross dependencies (debuguniforms.h). --- renderdoc/data/glsl/debuguniforms.h | 9 +-- renderdoc/data/glsl/mesh.comp | 84 ++++++++++++++++++++++------ renderdoc/driver/gl/gl_debug.cpp | 47 ++++++++++++---- renderdoc/driver/vulkan/vk_debug.cpp | 3 + 4 files changed, 107 insertions(+), 36 deletions(-) diff --git a/renderdoc/data/glsl/debuguniforms.h b/renderdoc/data/glsl/debuguniforms.h index 12fabb0ce..d4a1cab09 100644 --- a/renderdoc/data/glsl/debuguniforms.h +++ b/renderdoc/data/glsl/debuguniforms.h @@ -132,15 +132,10 @@ INST_NAME(general); BINDING(0) uniform MeshPickUBOData { - vec2 coords; - vec2 viewport; - - mat4 mvp; - + vec3 rayPos; uint use_indices; + vec3 rayDir; uint numVerts; - uint unproject; - uint padding; } INST_NAME(meshpick); diff --git a/renderdoc/data/glsl/mesh.comp b/renderdoc/data/glsl/mesh.comp index 9d994a957..107484a37 100644 --- a/renderdoc/data/glsl/mesh.comp +++ b/renderdoc/data/glsl/mesh.comp @@ -40,37 +40,87 @@ layout(binding = 3, std140) buffer pickresult_buffer layout (local_size_x = 128, local_size_y = 1, local_size_z = 1) in; +bool TriangleRayIntersect(vec3 A, vec3 B, vec3 C, + vec3 RayPosition, vec3 RayDirection, out vec3 HitPosition) +{ + bool Result = false; + + vec3 v0v1 = B - A; + vec3 v0v2 = C - A; + vec3 pvec = cross(RayDirection, v0v2); + float det = dot(v0v1, pvec); + + // if the determinant is negative the triangle is backfacing + // if the determinant is close to 0, the ray misses the triangle + if (abs(det) > 0.000001f) + { + float invDet = 1 / det; + + vec3 tvec = RayPosition - A; + vec3 qvec = cross(tvec, v0v1); + float u = dot(tvec, pvec) * invDet; + float v = dot(RayDirection, qvec) * invDet; + + if (u > 0 && u < 1 && + v > 0 && u + v < 1) + { + float t = dot(v0v2, qvec) * invDet; + HitPosition = RayPosition + (RayDirection * t); + //Info->Position = HitPosition; + Result = true; + } + } + + return Result; +} + void main() { uvec3 tid = gl_GlobalInvocationID; - uint vertid = tid.x; + uint vertid = tid.x * 3; - if(vertid >= meshpick.numVerts) - return; + // NOTE(james): when I turn on following early out the shader doesn't do anything. + // so I end up with a lot of duplicated work on smaller meshes - uint idx = meshpick.use_indices != 0u ? ib.data[vertid] : vertid; + //if(vertid >= meshpick.numVerts) + // return; + vertid = uint(mod(float(vertid), float(meshpick.numVerts))); + - vec4 pos = vb.data[idx]; + uint idx0 = meshpick.use_indices != 0u ? ib.data[vertid ] : vertid; + uint idx1 = meshpick.use_indices != 0u ? ib.data[vertid+1] : vertid+1; + uint idx2 = meshpick.use_indices != 0u ? ib.data[vertid+2] : vertid+2; + + vec4 pos0 = vb.data[idx0]; + vec4 pos1 = vb.data[idx1]; + vec4 pos2 = vb.data[idx2]; #ifdef VULKAN if(meshpick.unproject == 1u) pos = vec4(pos.x, -pos.y, pos.z, pos.w); #endif - vec4 wpos = meshpick.mvp * pos; - - wpos.xyz /= wpos.www; - - wpos.xy *= vec2(1.0f, -1.0f); - - vec2 scr = (wpos.xy + 1.0f) * 0.5f * meshpick.viewport; - - // close to target co-ords? add to list - float len = length(scr - meshpick.coords); - if(len < 35.0f) + vec3 hitPosition; + bool hit = TriangleRayIntersect(pos0.xyz, pos1.xyz, pos2.xyz, + meshpick.rayPos, meshpick.rayDir, hitPosition); + //push every vert of the triangle if its hit + if (hit) { + float dist0 = distance(pos0.xyz, hitPosition); + float dist1 = distance(pos1.xyz, hitPosition); + float dist2 = distance(pos2.xyz, hitPosition); + uint result_idx = atomicAdd(pickresult.counter.x, 1u); - pickresult.results[result_idx] = uvec4(vertid, idx, floatBitsToUint(len), floatBitsToUint(wpos.z)); + if (dist1 < dist0 && dist1 < dist2) + { + vertid +=1; + } + else if (dist2 < dist0 && dist2 < dist1) + { + vertid +=2; + } + pickresult.results[result_idx] = uvec4(vertid, + floatBitsToUint(hitPosition.x), floatBitsToUint(hitPosition.y), floatBitsToUint(hitPosition.z)); } } diff --git a/renderdoc/driver/gl/gl_debug.cpp b/renderdoc/driver/gl/gl_debug.cpp index 1a7ca60b7..1607fde34 100644 --- a/renderdoc/driver/gl/gl_debug.cpp +++ b/renderdoc/driver/gl/gl_debug.cpp @@ -1045,17 +1045,39 @@ uint32_t GLReplay::PickVertex(uint32_t eventID, const MeshDisplay &cfg, uint32_t PickMVP = projMat.Mul(camMat.Mul(guessProj.Inverse())); } + vec3 rayPos; + vec3 rayDir; + // convert mouse pos to world space ray + { + Matrix4f InversePickMVP = PickMVP.Inverse(); + + float pickX = ((float)x) / ((float)DebugData.outWidth); + float pickXCanonical = RDCLERP(-1.0f, 1.0f, pickX); + + float pickY = ((float)y) / ((float)DebugData.outHeight); + // flip the Y axis + float pickYCanonical = RDCLERP(1.0f, -1.0f, pickY); + + vec3 CameraToWorldNearPosition = + InversePickMVP.Transform(Vec3f(pickXCanonical, pickYCanonical, -1), 1); + vec3 CameraToWorldFarPosition = + InversePickMVP.Transform(Vec3f(pickXCanonical, pickYCanonical, 1), 1); + rayDir = (CameraToWorldFarPosition - CameraToWorldNearPosition); + rayDir.Normalise(); + rayPos = CameraToWorldNearPosition; + } + gl.glBindBufferBase(eGL_UNIFORM_BUFFER, 0, DebugData.UBOs[0]); MeshPickUBOData *cdata = (MeshPickUBOData *)gl.glMapBufferRange(eGL_UNIFORM_BUFFER, 0, sizeof(MeshPickUBOData), GL_MAP_WRITE_BIT | GL_MAP_INVALIDATE_BUFFER_BIT); - cdata->coords = Vec2f((float)x, (float)y); - cdata->viewport = Vec2f(DebugData.outWidth, DebugData.outHeight); + cdata->rayPos = rayPos; + cdata->rayDir = rayDir; cdata->use_indices = cfg.position.idxByteWidth ? 1U : 0U; cdata->numVerts = cfg.position.numVerts; - cdata->unproject = cfg.position.unproject; - cdata->mvp = PickMVP; + // cdata->unproject = cfg.position.unproject; + // cdata->mvp = PickMVP; gl.glUnmapBuffer(eGL_UNIFORM_BUFFER); @@ -1165,7 +1187,8 @@ uint32_t GLReplay::PickVertex(uint32_t eventID, const MeshDisplay &cfg, uint32_t (GLsizeiptr)(cfg.position.idxoffs + cfg.position.idxByteWidth * cfg.position.numVerts)); gl.glBindBufferBase(eGL_SHADER_STORAGE_BUFFER, 3, DebugData.pickResultBuf); - gl.glDispatchCompute(GLuint(cfg.position.numVerts / 128 + 1), 1, 1); + gl.glDispatchCompute(GLuint((cfg.position.numVerts / 3) / 128 + 1), 1, + 1); // launch one thread per triangle gl.glMemoryBarrier(GL_ATOMIC_COUNTER_BARRIER_BIT | GL_SHADER_STORAGE_BARRIER_BIT); uint32_t numResults = 0; @@ -1178,9 +1201,7 @@ uint32_t GLReplay::PickVertex(uint32_t eventID, const MeshDisplay &cfg, uint32_t struct PickResult { uint32_t vertid; - uint32_t idx; - float len; - float depth; + vec3 intersectionPoint; }; byte *mapped = (byte *)gl.glMapNamedBufferEXT(DebugData.pickResultBuf, eGL_READ_ONLY); @@ -1190,6 +1211,8 @@ uint32_t GLReplay::PickVertex(uint32_t eventID, const MeshDisplay &cfg, uint32_t PickResult *pickResults = (PickResult *)mapped; PickResult *closest = pickResults; + // distance from raycast hit to nearest worldspace position of the mouse + float closestPickDistance = (closest->intersectionPoint - rayPos).Length(); // min with size of results buffer to protect against overflows for(uint32_t i = 1; i < RDCMIN((uint32_t)DebugRenderData::maxMeshPicks, numResults); i++) @@ -1201,11 +1224,11 @@ uint32_t GLReplay::PickVertex(uint32_t eventID, const MeshDisplay &cfg, uint32_t // We could do something to try and disambiguate, but it's // never going to be intuitive, it's just going to flicker // confusingly. - if(pickResults[i].len < closest->len || - (pickResults[i].len == closest->len && pickResults[i].depth < closest->depth) || - (pickResults[i].len == closest->len && pickResults[i].depth == closest->depth && - pickResults[i].vertid < closest->vertid)) + float pickDistance = (pickResults[i].intersectionPoint - rayPos).Length(); + if(pickDistance < closestPickDistance) + { closest = pickResults + i; + } } uint32_t ret = closest->vertid; diff --git a/renderdoc/driver/vulkan/vk_debug.cpp b/renderdoc/driver/vulkan/vk_debug.cpp index 71eaff21d..753e82272 100644 --- a/renderdoc/driver/vulkan/vk_debug.cpp +++ b/renderdoc/driver/vulkan/vk_debug.cpp @@ -3032,6 +3032,7 @@ FloatVector VulkanDebugManager::InterpretVertex(byte *data, uint32_t vert, const uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg, uint32_t x, uint32_t y, uint32_t w, uint32_t h) { +#if 0 VkDevice dev = m_pDriver->GetDev(); const VkLayerDispatchTable *vt = ObjDisp(dev); @@ -3316,6 +3317,8 @@ uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg m_MeshPickResultReadback.Unmap(); return ret; +#endif + return 0; } void VulkanDebugManager::EndText(const TextPrintState &textstate)