lee/renderdoc
1
0
Fork 0
mirror of https://github.com/baldurk/renderdoc.git synced 2026-09-24 14:45:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reduce stack size allocated for SerialiseNext() in debug builds

Browse Source 
* MSVC gives each variable in the switch/case its own stack space, which means
  this function needs a bunch and if we serialise a long pNext chain (like
  device features) we can easily overflow the stack.
* I didn't check release, my suspicion is that this isn't an issue there, but
  the workaround is small so it's still useful to do.
This commit is contained in:
baldurk
2021-10-26 12:36:34 +01:00
parent 56ba5a686c
commit 2cb612f451
1 changed files with 23 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
renderdoc/driver/vulkan/vk_serialise.cpp
+23 -17
View File
@@ -1497,6 +1497,9 @@ SERIALISE_VK_HANDLES();
PNEXT_UNSUPPORTED(VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MUTABLE_DESCRIPTOR_TYPE_FEATURES_VALVE) \
PNEXT_UNSUPPORTED(VK_STRUCTURE_TYPE_MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE)
static const rdcliteral pNextName = "pNext"_lit;
static const rdcliteral pNextTypeName = "pNextType"_lit;
template <typename SerialiserType>
static void SerialiseNext(SerialiserType &ser, VkStructureType &sType, const void *&pNext)
{
@@ -1538,20 +1541,21 @@ static void SerialiseNext(SerialiserType &ser, VkStructureType &sType, const voi
}
// if we come across a struct we should process, then serialise a pointer to it.
#define PNEXT_STRUCT(StructType, StructName) \
case StructType: \
{ \
StructName *nextStruct = NULL; \
ser.SerialiseNullable("pNext"_lit, nextStruct); \
pNext = nextStruct; \
handled = true; \
break; \
#define PNEXT_STRUCT(StructType, StructName) \
case StructType: \
{ \
ser.SerialiseNullable(pNextName, (StructName *&)nextStruct); \
pNext = nextStruct; \
handled = true; \
break; \
}
// we don't want a default case to ensure we get a compile error if we forget to implement a
// structure type, but we also want to error if the input is invalid, so have this flag here.
bool handled = false;
void *nextStruct = NULL;
// this serialises the pNext with the right type, as nullable. We already know from above that
// there IS something here, so the nullable is redundant but convenient
switch(*nextType)
@@ -1588,15 +1592,15 @@ static void SerialiseNext(SerialiserType &ser, VkStructureType &sType, const voi
// We don't have to go any further, the act of serialising this struct will walk the chain further,
// so we can return immediately.
#undef PNEXT_STRUCT
#define PNEXT_STRUCT(StructType, StructName) \
case StructType: \
{ \
VkStructureType *nextType = &next->sType; \
ser.SerialiseNullable("pNextType"_lit, nextType); \
StructName *actual = (StructName *)next; \
ser.SerialiseNullable("pNext"_lit, actual); \
handled = true; \
return; \
#define PNEXT_STRUCT(StructType, StructName) \
case StructType: \
{ \
nextType = &next->sType; \
ser.SerialiseNullable(pNextTypeName, nextType); \
actualStruct = (void *)next; \
ser.SerialiseNullable(pNextName, (StructName *&)actualStruct); \
handled = true; \
return; \
}
// we don't want a default case to ensure we get a compile error if we forget to implement a
@@ -1605,6 +1609,8 @@ static void SerialiseNext(SerialiserType &ser, VkStructureType &sType, const voi
// walk the pNext chain, skipping any structs we don't care about serialising.
VkBaseInStructure *next = (VkBaseInStructure *)pNext;
VkStructureType *nextType = NULL;
void *actualStruct = NULL;
while(next)
{