From a08455c4fb4181221fa62ba520e9b3ed2e59d14c Mon Sep 17 00:00:00 2001 From: baldurk Date: Wed, 23 Oct 2019 11:23:32 +0100 Subject: [PATCH] Don't crash if vertex buffer stride is 0 when picking vertices --- qrenderdoc/Windows/BufferViewer.cpp | 4 ++-- renderdoc/driver/d3d11/d3d11_replay.cpp | 2 +- renderdoc/driver/d3d12/d3d12_replay.cpp | 2 +- renderdoc/driver/gl/gl_debug.cpp | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/qrenderdoc/Windows/BufferViewer.cpp b/qrenderdoc/Windows/BufferViewer.cpp index ab96e33fe..e69f6388c 100644 --- a/qrenderdoc/Windows/BufferViewer.cpp +++ b/qrenderdoc/Windows/BufferViewer.cpp @@ -1385,8 +1385,8 @@ static void ConfigureMeshColumns(ICaptureContext &ctx, PopulateBufferData *bufda BufferDescription *buf = ctx.GetBuffer(vb.resourceId); if(buf) { - numRowsUpperBound = - qMax(numRowsUpperBound, uint32_t(buf->length - vb.byteOffset) / vb.byteStride); + numRowsUpperBound = qMax(numRowsUpperBound, + uint32_t(buf->length - vb.byteOffset) / qMax(1U, vb.byteStride)); } } diff --git a/renderdoc/driver/d3d11/d3d11_replay.cpp b/renderdoc/driver/d3d11/d3d11_replay.cpp index ad4611236..d3ec62f70 100644 --- a/renderdoc/driver/d3d11/d3d11_replay.cpp +++ b/renderdoc/driver/d3d11/d3d11_replay.cpp @@ -2941,7 +2941,7 @@ uint32_t D3D11Replay::PickVertex(uint32_t eventId, int32_t width, int32_t height GetDebugManager()->GetBufferData(vb, cfg.position.vertexByteOffset, 0, oldData); // clamp maxIndex to upper bound in case we got invalid indices or primitive restart indices - maxIndex = RDCMIN(maxIndex, uint32_t(oldData.size() / cfg.position.vertexByteStride)); + maxIndex = RDCMIN(maxIndex, uint32_t(oldData.size() / RDCMAX(1U, cfg.position.vertexByteStride))); if(m_VertexPick.PickVBBuf == NULL || m_VertexPick.PickVBSize < (maxIndex + 1) * sizeof(Vec4f)) { diff --git a/renderdoc/driver/d3d12/d3d12_replay.cpp b/renderdoc/driver/d3d12/d3d12_replay.cpp index c7692851c..7632ca91d 100644 --- a/renderdoc/driver/d3d12/d3d12_replay.cpp +++ b/renderdoc/driver/d3d12/d3d12_replay.cpp @@ -2075,7 +2075,7 @@ uint32_t D3D12Replay::PickVertex(uint32_t eventId, int32_t width, int32_t height GetDebugManager()->GetBufferData(vb, cfg.position.vertexByteOffset, 0, oldData); // clamp maxIndex to upper bound in case we got invalid indices or primitive restart indices - maxIndex = RDCMIN(maxIndex, uint32_t(oldData.size() / cfg.position.vertexByteStride)); + maxIndex = RDCMIN(maxIndex, uint32_t(oldData.size() / RDCMAX(1U, cfg.position.vertexByteStride))); if(vb) { diff --git a/renderdoc/driver/gl/gl_debug.cpp b/renderdoc/driver/gl/gl_debug.cpp index dbc75ebf3..4064f8831 100644 --- a/renderdoc/driver/gl/gl_debug.cpp +++ b/renderdoc/driver/gl/gl_debug.cpp @@ -2128,7 +2128,7 @@ uint32_t GLReplay::PickVertex(uint32_t eventId, int32_t width, int32_t height, GetBufferData(cfg.position.vertexResourceId, cfg.position.vertexByteOffset, 0, oldData); // clamp maxIndex to upper bound in case we got invalid indices or primitive restart indices - maxIndex = RDCMIN(maxIndex, uint32_t(oldData.size() / cfg.position.vertexByteStride)); + maxIndex = RDCMIN(maxIndex, uint32_t(oldData.size() / RDCMAX(1U, cfg.position.vertexByteStride))); if(DebugData.pickVBBuf == 0 || DebugData.pickVBSize < (maxIndex + 1) * sizeof(Vec4f)) {