diff --git a/renderdoc/data/glsl/debuguniforms.h b/renderdoc/data/glsl/debuguniforms.h index 12fabb0ce..d4a1cab09 100644 --- a/renderdoc/data/glsl/debuguniforms.h +++ b/renderdoc/data/glsl/debuguniforms.h @@ -132,15 +132,10 @@ INST_NAME(general); BINDING(0) uniform MeshPickUBOData { - vec2 coords; - vec2 viewport; - - mat4 mvp; - + vec3 rayPos; uint use_indices; + vec3 rayDir; uint numVerts; - uint unproject; - uint padding; } INST_NAME(meshpick); diff --git a/renderdoc/data/glsl/mesh.comp b/renderdoc/data/glsl/mesh.comp index 9d994a957..107484a37 100644 --- a/renderdoc/data/glsl/mesh.comp +++ b/renderdoc/data/glsl/mesh.comp @@ -40,37 +40,87 @@ layout(binding = 3, std140) buffer pickresult_buffer layout (local_size_x = 128, local_size_y = 1, local_size_z = 1) in; +bool TriangleRayIntersect(vec3 A, vec3 B, vec3 C, + vec3 RayPosition, vec3 RayDirection, out vec3 HitPosition) +{ + bool Result = false; + + vec3 v0v1 = B - A; + vec3 v0v2 = C - A; + vec3 pvec = cross(RayDirection, v0v2); + float det = dot(v0v1, pvec); + + // if the determinant is negative the triangle is backfacing + // if the determinant is close to 0, the ray misses the triangle + if (abs(det) > 0.000001f) + { + float invDet = 1 / det; + + vec3 tvec = RayPosition - A; + vec3 qvec = cross(tvec, v0v1); + float u = dot(tvec, pvec) * invDet; + float v = dot(RayDirection, qvec) * invDet; + + if (u > 0 && u < 1 && + v > 0 && u + v < 1) + { + float t = dot(v0v2, qvec) * invDet; + HitPosition = RayPosition + (RayDirection * t); + //Info->Position = HitPosition; + Result = true; + } + } + + return Result; +} + void main() { uvec3 tid = gl_GlobalInvocationID; - uint vertid = tid.x; + uint vertid = tid.x * 3; - if(vertid >= meshpick.numVerts) - return; + // NOTE(james): when I turn on following early out the shader doesn't do anything. + // so I end up with a lot of duplicated work on smaller meshes - uint idx = meshpick.use_indices != 0u ? ib.data[vertid] : vertid; + //if(vertid >= meshpick.numVerts) + // return; + vertid = uint(mod(float(vertid), float(meshpick.numVerts))); + - vec4 pos = vb.data[idx]; + uint idx0 = meshpick.use_indices != 0u ? ib.data[vertid ] : vertid; + uint idx1 = meshpick.use_indices != 0u ? ib.data[vertid+1] : vertid+1; + uint idx2 = meshpick.use_indices != 0u ? ib.data[vertid+2] : vertid+2; + + vec4 pos0 = vb.data[idx0]; + vec4 pos1 = vb.data[idx1]; + vec4 pos2 = vb.data[idx2]; #ifdef VULKAN if(meshpick.unproject == 1u) pos = vec4(pos.x, -pos.y, pos.z, pos.w); #endif - vec4 wpos = meshpick.mvp * pos; - - wpos.xyz /= wpos.www; - - wpos.xy *= vec2(1.0f, -1.0f); - - vec2 scr = (wpos.xy + 1.0f) * 0.5f * meshpick.viewport; - - // close to target co-ords? add to list - float len = length(scr - meshpick.coords); - if(len < 35.0f) + vec3 hitPosition; + bool hit = TriangleRayIntersect(pos0.xyz, pos1.xyz, pos2.xyz, + meshpick.rayPos, meshpick.rayDir, hitPosition); + //push every vert of the triangle if its hit + if (hit) { + float dist0 = distance(pos0.xyz, hitPosition); + float dist1 = distance(pos1.xyz, hitPosition); + float dist2 = distance(pos2.xyz, hitPosition); + uint result_idx = atomicAdd(pickresult.counter.x, 1u); - pickresult.results[result_idx] = uvec4(vertid, idx, floatBitsToUint(len), floatBitsToUint(wpos.z)); + if (dist1 < dist0 && dist1 < dist2) + { + vertid +=1; + } + else if (dist2 < dist0 && dist2 < dist1) + { + vertid +=2; + } + pickresult.results[result_idx] = uvec4(vertid, + floatBitsToUint(hitPosition.x), floatBitsToUint(hitPosition.y), floatBitsToUint(hitPosition.z)); } } diff --git a/renderdoc/driver/gl/gl_debug.cpp b/renderdoc/driver/gl/gl_debug.cpp index 1a7ca60b7..1607fde34 100644 --- a/renderdoc/driver/gl/gl_debug.cpp +++ b/renderdoc/driver/gl/gl_debug.cpp @@ -1045,17 +1045,39 @@ uint32_t GLReplay::PickVertex(uint32_t eventID, const MeshDisplay &cfg, uint32_t PickMVP = projMat.Mul(camMat.Mul(guessProj.Inverse())); } + vec3 rayPos; + vec3 rayDir; + // convert mouse pos to world space ray + { + Matrix4f InversePickMVP = PickMVP.Inverse(); + + float pickX = ((float)x) / ((float)DebugData.outWidth); + float pickXCanonical = RDCLERP(-1.0f, 1.0f, pickX); + + float pickY = ((float)y) / ((float)DebugData.outHeight); + // flip the Y axis + float pickYCanonical = RDCLERP(1.0f, -1.0f, pickY); + + vec3 CameraToWorldNearPosition = + InversePickMVP.Transform(Vec3f(pickXCanonical, pickYCanonical, -1), 1); + vec3 CameraToWorldFarPosition = + InversePickMVP.Transform(Vec3f(pickXCanonical, pickYCanonical, 1), 1); + rayDir = (CameraToWorldFarPosition - CameraToWorldNearPosition); + rayDir.Normalise(); + rayPos = CameraToWorldNearPosition; + } + gl.glBindBufferBase(eGL_UNIFORM_BUFFER, 0, DebugData.UBOs[0]); MeshPickUBOData *cdata = (MeshPickUBOData *)gl.glMapBufferRange(eGL_UNIFORM_BUFFER, 0, sizeof(MeshPickUBOData), GL_MAP_WRITE_BIT | GL_MAP_INVALIDATE_BUFFER_BIT); - cdata->coords = Vec2f((float)x, (float)y); - cdata->viewport = Vec2f(DebugData.outWidth, DebugData.outHeight); + cdata->rayPos = rayPos; + cdata->rayDir = rayDir; cdata->use_indices = cfg.position.idxByteWidth ? 1U : 0U; cdata->numVerts = cfg.position.numVerts; - cdata->unproject = cfg.position.unproject; - cdata->mvp = PickMVP; + // cdata->unproject = cfg.position.unproject; + // cdata->mvp = PickMVP; gl.glUnmapBuffer(eGL_UNIFORM_BUFFER); @@ -1165,7 +1187,8 @@ uint32_t GLReplay::PickVertex(uint32_t eventID, const MeshDisplay &cfg, uint32_t (GLsizeiptr)(cfg.position.idxoffs + cfg.position.idxByteWidth * cfg.position.numVerts)); gl.glBindBufferBase(eGL_SHADER_STORAGE_BUFFER, 3, DebugData.pickResultBuf); - gl.glDispatchCompute(GLuint(cfg.position.numVerts / 128 + 1), 1, 1); + gl.glDispatchCompute(GLuint((cfg.position.numVerts / 3) / 128 + 1), 1, + 1); // launch one thread per triangle gl.glMemoryBarrier(GL_ATOMIC_COUNTER_BARRIER_BIT | GL_SHADER_STORAGE_BARRIER_BIT); uint32_t numResults = 0; @@ -1178,9 +1201,7 @@ uint32_t GLReplay::PickVertex(uint32_t eventID, const MeshDisplay &cfg, uint32_t struct PickResult { uint32_t vertid; - uint32_t idx; - float len; - float depth; + vec3 intersectionPoint; }; byte *mapped = (byte *)gl.glMapNamedBufferEXT(DebugData.pickResultBuf, eGL_READ_ONLY); @@ -1190,6 +1211,8 @@ uint32_t GLReplay::PickVertex(uint32_t eventID, const MeshDisplay &cfg, uint32_t PickResult *pickResults = (PickResult *)mapped; PickResult *closest = pickResults; + // distance from raycast hit to nearest worldspace position of the mouse + float closestPickDistance = (closest->intersectionPoint - rayPos).Length(); // min with size of results buffer to protect against overflows for(uint32_t i = 1; i < RDCMIN((uint32_t)DebugRenderData::maxMeshPicks, numResults); i++) @@ -1201,11 +1224,11 @@ uint32_t GLReplay::PickVertex(uint32_t eventID, const MeshDisplay &cfg, uint32_t // We could do something to try and disambiguate, but it's // never going to be intuitive, it's just going to flicker // confusingly. - if(pickResults[i].len < closest->len || - (pickResults[i].len == closest->len && pickResults[i].depth < closest->depth) || - (pickResults[i].len == closest->len && pickResults[i].depth == closest->depth && - pickResults[i].vertid < closest->vertid)) + float pickDistance = (pickResults[i].intersectionPoint - rayPos).Length(); + if(pickDistance < closestPickDistance) + { closest = pickResults + i; + } } uint32_t ret = closest->vertid; diff --git a/renderdoc/driver/vulkan/vk_debug.cpp b/renderdoc/driver/vulkan/vk_debug.cpp index 71eaff21d..753e82272 100644 --- a/renderdoc/driver/vulkan/vk_debug.cpp +++ b/renderdoc/driver/vulkan/vk_debug.cpp @@ -3032,6 +3032,7 @@ FloatVector VulkanDebugManager::InterpretVertex(byte *data, uint32_t vert, const uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg, uint32_t x, uint32_t y, uint32_t w, uint32_t h) { +#if 0 VkDevice dev = m_pDriver->GetDev(); const VkLayerDispatchTable *vt = ObjDisp(dev); @@ -3316,6 +3317,8 @@ uint32_t VulkanDebugManager::PickVertex(uint32_t eventID, const MeshDisplay &cfg m_MeshPickResultReadback.Unmap(); return ret; +#endif + return 0; } void VulkanDebugManager::EndText(const TextPrintState &textstate)