lee/renderdoc
1
0
Fork 0
mirror of https://github.com/baldurk/renderdoc.git synced 2026-09-22 05:35:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement proper out-of-bounds value filling for vulkan post-vs fetch

Browse Source
This commit is contained in:
baldurk
2021-04-23 19:07:31 +01:00
parent 659c75319b
commit e78a48bfc3
3 changed files with 43 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
renderdoc/replay/replay_driver.cpp
+8
View File
@@ -587,6 +587,10 @@ FloatVector HighlightCache::InterpretVertex(const byte *data, uint32_t vert, con
{
FloatVector ret(0.0f, 0.0f, 0.0f, 1.0f);
if(cfg.position.format.compType == CompType::UInt ||
cfg.position.format.compType == CompType::SInt || cfg.position.format.compCount == 4)
ret.w = 0.0f;
if(useidx && idxData)
{
if(vert >= (uint32_t)indices.size())
@@ -623,6 +627,10 @@ FloatVector HighlightCache::InterpretVertex(const byte *data, uint32_t vert,
if(data + fmt.ElementSize() > end)
{
valid = false;
if(fmt.compType == CompType::UInt || fmt.compType == CompType::SInt || fmt.compCount == 4)
return FloatVector(0.0f, 0.0f, 0.0f, 0.0f);
return FloatVector(0.0f, 0.0f, 0.0f, 1.0f);
}