Support lazy-generating structured data objects for large arrays

* For certain very large arrays it can be nice to defer generation of structured
  data until it's needed, since often maybe only a handful of elements may be
  needed (or commonly none at all).

qrenderdoc/Windows/Dialogs/ConfigEditor.cpp

@@ -59,8 +59,8 @@ static QString valueString(const SDObject *o)
 
 static bool anyChildChanged(const SDObject *o)
 {
-  SDObject *def = o->FindChild("default");
-  SDObject *val = o->FindChild("value");
+  const SDObject *def = o->FindChild("default");
+  const SDObject *val = o->FindChild("value");
 
   if(val && def)
     return !val->HasEqualValue(def);

qrenderdoc/Windows/Dialogs/SettingsDialog.cpp

@@ -494,7 +494,7 @@ void SettingsDialog::on_chooseSearchPaths_clicked()
 
   QStringList items;
 
-  for(SDObject *c : *getPaths)
+  for(const SDObject *c : *getPaths)
     items << c->data.str;
 
   list.setItems(items);

renderdoc/api/replay/structured_data.h

@@ -25,6 +25,7 @@
 #pragma once
 
 #include <stdint.h>
+#include <functional>
 #include "apidefs.h"
 #include "rdcarray.h"
 #include "rdcstr.h"
@@ -339,6 +340,11 @@ private:
 
 DECLARE_REFLECTION_STRUCT(StructuredObjectList);
 
+// due to some objects potentially being lazily generated, we use the ugly 'mutable' keyword here
+// to avoid completely losing const on these objects but allowing us to actually modify objects
+// behind the scenes inside const objects. This is only used for effectively caching the lazy
+// generated results, so to the outside world the object is still const.
+
 DOCUMENT("The data inside an :class:`SDObject` whether it's plain old data or complex children.");
 struct SDObjectData
 {
@@ -365,7 +371,7 @@ private:
   friend void DoSerialise(SerialiserType &ser, SDObject *el);
 
   DOCUMENT("A list of :class:`SDObject` containing the children of this :class:`SDObject`.");
-  StructuredObjectList children;
+  mutable StructuredObjectList children;
 
   void *operator new(size_t count) = delete;
   void *operator new[](size_t count) = delete;
@@ -375,6 +381,17 @@ private:
 
 DECLARE_REFLECTION_STRUCT(SDObjectData);
 
+#if !defined(SWIG)
+using LazyGenerator = std::function<SDObject *(const void *)>;
+
+struct LazyArrayData
+{
+  byte *data;
+  size_t elemSize;
+  LazyGenerator generator;
+};
+#endif
+
 DOCUMENT(R"(Defines a single structured object. Structured objects are defined recursively and one
 object can either be a basic type (integer, float, etc), an array, or a struct. Arrays and structs
 are defined similarly.
@@ -384,28 +401,70 @@ Each object owns its children and they will be deleted when it is deleted. You c
 )");
 struct SDObject
 {
+#if !defined(SWIG)
+  template <typename MaybeConstSDObject>
+  struct SDObjectIt
+  {
+  private:
+    MaybeConstSDObject *o;
+    size_t i;
+
+  public:
+    using iterator_category = std::bidirectional_iterator_tag;
+    using value_type = MaybeConstSDObject *;
+    using difference_type = ptrdiff_t;
+    using pointer = value_type *;
+    using reference = value_type &;
+
+    SDObjectIt(MaybeConstSDObject *obj, size_t index) : o(obj), i(index) {}
+    SDObjectIt(const SDObjectIt &rhs) : o(rhs.o), i(rhs.i) {}
+    SDObjectIt &operator++()
+    {
+      ++i;
+      return *this;
+    }
+    SDObjectIt operator++(int)
+    {
+      SDObjectIt tmp(*this);
+      operator++();
+      return tmp;
+    }
+    SDObjectIt &operator--()
+    {
+      --i;
+      return *this;
+    }
+    SDObjectIt operator--(int)
+    {
+      SDObjectIt tmp(*this);
+      operator--();
+      return tmp;
+    }
+    size_t operator-(const SDObjectIt &rhs) { return i - rhs.i; }
+    SDObjectIt operator+(int shift)
+    {
+      SDObjectIt ret(*this);
+      ret.i += shift;
+      return ret;
+    }
+    bool operator==(const SDObjectIt &rhs) { return o == rhs.o && i == rhs.i; }
+    bool operator!=(const SDObjectIt &rhs) { return !(*this == rhs); }
+    SDObjectIt &operator=(const SDObjectIt &rhs)
+    {
+      o = rhs.o;
+      i = rhs.i;
+      return *this;
+    }
+
+    inline MaybeConstSDObject *operator*() const { return o->GetChild(i); }
+    inline MaybeConstSDObject &operator->() const { return *o->GetChild(i); }
+  };
+#endif
+
   /////////////////////////////////////////////////////////////////
   // memory management, in a dll safe way
-  void *operator new(size_t sz)
-  {
-    void *ret = NULL;
-#ifdef RENDERDOC_EXPORTS
-    ret = malloc(sz);
-    if(ret == NULL)
-      RENDERDOC_OutOfMemory(sz);
-#else
-    ret = RENDERDOC_AllocArrayMem(sz);
-#endif
-    return ret;
-  }
-  void operator delete(void *p)
-  {
-#ifdef RENDERDOC_EXPORTS
-    free(p);
-#else
-    RENDERDOC_FreeArrayMem(p);
-#endif
-  }
+  void *operator new(size_t sz) { return SDObject::alloc(sz); }
+  void operator delete(void *p) { SDObject::dealloc(p); }
   void *operator new[](size_t count) = delete;
   void operator delete[](void *p) = delete;
 
@@ -413,12 +472,16 @@ struct SDObject
   {
     name = n;
     data.basic.u = 0;
+    m_Lazy = NULL;
   }
 
   ~SDObject()
   {
     // we own our children, so delete them now.
     DeleteChildren();
+
+    // delete the lazy array data if we used it (rare)
+    DeleteLazyGenerator();
   }
 
   DOCUMENT("Create a deep copy of this object.");
@@ -430,6 +493,11 @@ struct SDObject
     ret->data.basic = data.basic;
     ret->data.str = data.str;
 
+    if(m_Lazy)
+    {
+      PopulateAllChildren();
+    }
+
     ret->data.children.resize(data.children.size());
     for(size_t i = 0; i < data.children.size(); i++)
       ret->data.children[i] = data.children[i]->Duplicate();
@@ -472,7 +540,10 @@ recursively through children.
     else
     {
       for(size_t c = 0; c < o->data.children.size(); c++)
-        ret &= data.children[c]->HasEqualValue(o->data.children[c]);
+      {
+        PopulateChild(c);
+        ret &= data.children[c]->HasEqualValue(o->GetChild(c));
+      }
     }
 
     return ret;
@@ -484,7 +555,14 @@ recursively through children.
 
 :param SDObject obj: The new child to add
 )");
-  inline void DuplicateAndAddChild(SDObject *child) { data.children.push_back(child->Duplicate()); }
+  inline void DuplicateAndAddChild(const SDObject *child)
+  {
+    // if we're adding to a lazy-generated array we can't have a mixture between lazy generation and
+    // fully owned children. This shouldn't happen, but just in case we'll evaluate the lazy array
+    // here.
+    PopulateAllChildren();
+    data.children.push_back(child->Duplicate());
+  }
   DOCUMENT(R"(Find a child object by a given name. If no matching child is found, ``None`` is
 returned.
 
@@ -492,11 +570,11 @@ returned.
 :return: A reference to the child object if found, or ``None`` if not.
 :rtype: SDObject
 )");
-  inline SDObject *FindChild(const rdcstr &childName) const
+  inline SDObject *FindChild(const rdcstr &childName)
   {
     for(size_t i = 0; i < data.children.size(); i++)
-      if(data.children[i]->name == childName)
-        return data.children[i];
+      if(GetChild(i)->name == childName)
+        return GetChild(i);
     return NULL;
   }
 
@@ -507,13 +585,38 @@ returned.
 :return: A reference to the child object if valid, or ``None`` if not.
 :rtype: SDObject
 )");
-  inline SDObject *GetChild(size_t index) const
+  inline SDObject *GetChild(size_t index)
   {
     if(index < data.children.size())
+    {
+      PopulateChild(index);
       return data.children[index];
+    }
+
     return NULL;
   }
 
+#if !defined(SWIG)
+  // const versions of FindChild/GetChild
+  inline const SDObject *FindChild(const rdcstr &childName) const
+  {
+    for(size_t i = 0; i < data.children.size(); i++)
+      if(GetChild(i)->name == childName)
+        return GetChild(i);
+    return NULL;
+  }
+  inline const SDObject *GetChild(size_t index) const
+  {
+    if(index < data.children.size())
+    {
+      PopulateChild(index);
+      return data.children[index];
+    }
+
+    return NULL;
+  }
+#endif
+
   DOCUMENT(R"(Delete the child object at an index. If the index is out of bounds, nothing happens.
 
 :param int index: The index to remove.
@@ -521,7 +624,12 @@ returned.
   inline void RemoveChild(size_t index)
   {
     if(index < data.children.size())
+    {
+      // we really shouldn't be deleting individually from a lazy array but just in case we are,
+      // fully evaluate it first.
+      PopulateAllChildren();
       delete data.children.takeAt(index);
+    }
   }
 
   DOCUMENT("Delete all child objects.");
@@ -531,6 +639,8 @@ returned.
       delete data.children[i];
 
     data.children.clear();
+
+    DeleteLazyGenerator();
   }
 
   DOCUMENT(R"(Get the number of child objects.
@@ -541,10 +651,13 @@ returned.
   inline size_t NumChildren() const { return data.children.size(); }
 #if !defined(SWIG)
   // these are for C++ iteration so not defined when SWIG is generating interfaces
-  inline SDObject *const *begin() const { return data.children.begin(); }
-  inline SDObject *const *end() const { return data.children.end(); }
-  inline SDObject **begin() { return data.children.begin(); }
-  inline SDObject **end() { return data.children.end(); }
+  inline SDObjectIt<const SDObject> begin() const { return SDObjectIt<const SDObject>(this, 0); }
+  inline SDObjectIt<const SDObject> end() const
+  {
+    return SDObjectIt<const SDObject>(this, data.children.size());
+  }
+  inline SDObjectIt<SDObject> begin() { return SDObjectIt<SDObject>(this, 0); }
+  inline SDObjectIt<SDObject> end() { return SDObjectIt<SDObject>(this, data.children.size()); }
 #endif
 
 #if !defined(SWIG)
@@ -556,21 +669,39 @@ returned.
   // immediately for easy chaining.
   SDObject *AddAndOwnChild(SDObject *child)
   {
+    PopulateAllChildren();
     data.children.push_back(child);
     return child;
   }
   // similar to AddAndOwnChild, but insert at a given offset
   SDObject *InsertAndOwnChild(size_t offs, SDObject *child)
   {
+    PopulateAllChildren();
     data.children.insert(offs, child);
     return child;
   }
   // Take ownership of the whole children array from the object.
   void TakeAllChildren(StructuredObjectList &objs)
   {
+    PopulateAllChildren();
     objs.clear();
     objs.swap(data.children);
   }
+
+  template <typename T>
+  void SetLazyArray(uint64_t arrayCount, T *arrayData, LazyGenerator generator)
+  {
+    DeleteChildren();
+
+    void *lazyAlloc = alloc(sizeof(LazyArrayData));
+
+    m_Lazy = new(lazyAlloc) LazyArrayData;
+    m_Lazy->generator = generator;
+    m_Lazy->elemSize = sizeof(T);
+    m_Lazy->data = (byte *)alloc(sizeof(T) * arrayCount);
+    memcpy(m_Lazy->data, arrayData, sizeof(T) * arrayCount);
+    data.children.resize(arrayCount);
+  }
 #endif
 
 // C++ gets more extensive typecasts. We'll add a couple for python in the interface file
@@ -709,6 +840,62 @@ protected:
   SDObject() {}
   SDObject(const SDObject &other) = delete;
   SDObject &operator=(const SDObject &other) = delete;
+
+  // these functions can be const because we have 'mutable' allowing us to modify these members.
+  // It's ugly, but necessary
+  inline void PopulateChild(size_t idx) const
+  {
+    if(m_Lazy)
+    {
+      if(data.children[idx] == NULL)
+        data.children[idx] = m_Lazy->generator(m_Lazy->data + idx * m_Lazy->elemSize);
+    }
+  }
+
+  void PopulateAllChildren() const
+  {
+    if(m_Lazy)
+    {
+      for(size_t i = 0; i < data.children.size(); i++)
+        PopulateChild(i);
+
+      DeleteLazyGenerator();
+    }
+  }
+
+  static void *alloc(size_t sz)
+  {
+    void *ret = NULL;
+#ifdef RENDERDOC_EXPORTS
+    ret = malloc(sz);
+    if(ret == NULL)
+      RENDERDOC_OutOfMemory(sz);
+#else
+    ret = RENDERDOC_AllocArrayMem(sz);
+#endif
+    return ret;
+  }
+  static void dealloc(void *p)
+  {
+#ifdef RENDERDOC_EXPORTS
+    free(p);
+#else
+    RENDERDOC_FreeArrayMem(p);
+#endif
+  }
+
+private:
+  mutable LazyArrayData *m_Lazy = NULL;
+
+  void DeleteLazyGenerator() const
+  {
+    if(m_Lazy)
+    {
+      dealloc(m_Lazy->data);
+      dealloc(m_Lazy);
+      m_Lazy = NULL;
+    }
+  }
 };
 
 DECLARE_REFLECTION_STRUCT(SDObject);
@@ -1002,6 +1189,9 @@ struct SDChunk : public SDObject
     ret->data.str = data.str;
 
     ret->data.children.resize(data.children.size());
+
+    PopulateAllChildren();
+
     for(size_t i = 0; i < data.children.size(); i++)
       ret->data.children[i] = data.children[i]->Duplicate();
 

renderdoc/core/settings.cpp

@@ -79,7 +79,7 @@ static SDObject *makeSDObject(const char *name, SDBasic type, pugi::xml_node &va
   return NULL;
 }
 
-static void saveSDObject(SDObject &value, pugi::xml_node obj)
+static void saveSDObject(const SDObject &value, pugi::xml_node obj)
 {
   switch(value.type.basetype)
   {
@@ -100,7 +100,7 @@ static void saveSDObject(SDObject &value, pugi::xml_node obj)
   }
 }
 
-static void Config2XML(pugi::xml_node &parent, SDObject &child)
+static void Config2XML(pugi::xml_node &parent, const SDObject &child)
 {
   pugi::xml_node obj = parent.append_child(child.name.c_str());
 
@@ -111,7 +111,7 @@ static void Config2XML(pugi::xml_node &parent, SDObject &child)
   }
   else
   {
-    SDObject *value = child.FindChild("value");
+    const SDObject *value = child.FindChild("value");
 
     parent.insert_child_before(pugi::node_comment, obj)
         .set_value((" " + child.FindChild("description")->data.str + " ").c_str());

renderdoc/driver/d3d12/d3d12_initstate.cpp

@@ -31,10 +31,6 @@
 #include "d3d12_manager.h"
 #include "d3d12_resources.h"
 
-RDOC_CONFIG(bool, D3D12_Debug_HideInitialDescriptors, false,
-            "Hide the initial contents of descriptor heaps. "
-            "For extremely large descriptor heaps this can drastically reduce memory consumption.");
-
 bool D3D12ResourceManager::Prepare_InitialState(ID3D12DeviceChild *res)
 {
   ResourceId id = GetResID(res);
@@ -430,16 +426,15 @@ bool D3D12ResourceManager::Serialise_InitialState(SerialiserType &ser, ResourceI
     D3D12Descriptor *Descriptors = initial ? initial->descriptors : NULL;
     uint32_t numElems = initial ? initial->numDescriptors : 0;
 
-    const bool hide = D3D12_Debug_HideInitialDescriptors();
-
-    if(hide)
-      ser.PushInternal();
+    // there's no point in setting up a lazy array when we're structured exporting because we KNOW
+    // we're going to need all the data anyway.
+    if(!IsStructuredExporting(m_State))
+      ser.SetLazyThreshold(1000);
 
     SERIALISE_ELEMENT_ARRAY(Descriptors, numElems);
     SERIALISE_ELEMENT(numElems);
 
-    if(hide)
-      ser.PopInternal();
+    ser.SetLazyThreshold(0);
 
     SERIALISE_CHECK_READ_ERRORS();
 

renderdoc/driver/vulkan/vk_initstate.cpp

@@ -36,10 +36,6 @@
 // command buffer that stalls the GPU).
 // See INITSTATEBATCH
 
-RDOC_CONFIG(bool, Vulkan_Debug_HideInitialDescriptors, false,
-            "Hide the initial contents of descriptor sets. "
-            "For extremely large descriptor sets this can drastically reduce memory consumption.");
-
 bool WrappedVulkan::Prepare_InitialState(WrappedVkRes *res)
 {
   ResourceId id = GetResourceManager()->GetID(res);
@@ -578,10 +574,10 @@ bool WrappedVulkan::Serialise_InitialState(SerialiserType &ser, ResourceId id, V
     uint32_t NumBindings = 0;
     bytebuf InlineData;
 
-    const bool hide = Vulkan_Debug_HideInitialDescriptors();
-
-    if(hide)
-      ser.PushInternal();
+    // there's no point in setting up a lazy array when we're structured exporting because we KNOW
+    // we're going to need all the data anyway.
+    if(!IsStructuredExporting(m_State))
+      ser.SetLazyThreshold(1000);
 
     // while writing, fetching binding information from prepared initial contents
     if(ser.IsWriting())
@@ -595,14 +591,13 @@ bool WrappedVulkan::Serialise_InitialState(SerialiserType &ser, ResourceId id, V
     SERIALISE_ELEMENT_ARRAY(Bindings, NumBindings);
     SERIALISE_ELEMENT(NumBindings);
 
+    ser.SetLazyThreshold(0);
+
     if(ser.VersionAtLeast(0x12))
     {
       SERIALISE_ELEMENT(InlineData);
     }
 
-    if(hide)
-      ser.PopInternal();
-
     SERIALISE_CHECK_READ_ERRORS();
 
     // while reading, fetch the binding information and allocate a VkWriteDescriptorSet array

renderdoc/renderdoc.vcxproj

@@ -573,7 +573,9 @@
     <ClCompile Include="common\threading_tests.cpp" />
     <ClCompile Include="core\bit_flag_iterator_tests.cpp" />
     <ClCompile Include="core\settings.cpp" />
-    <ClCompile Include="core\core.cpp" />
+    <ClCompile Include="core\core.cpp">
+      <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions>
+    </ClCompile>
     <ClCompile Include="core\image_viewer.cpp" />
     <ClCompile Include="core\intervals_tests.cpp" />
     <ClCompile Include="core\plugins.cpp" />

renderdoc/serialise/serialiser.cpp

@@ -602,7 +602,7 @@ void Serialiser<SerialiserMode::Writing>::WriteStructuredFile(const SDFile &file
       for(size_t o = 0; o < chunk.NumChildren(); o++)
       {
         // note, we don't need names because we aren't exporting structured data
-        ser->Serialise(""_lit, chunk.GetChild(o));
+        ser->Serialise(""_lit, (SDObject *)chunk.GetChild(o));
       }
     }
 
@@ -755,7 +755,15 @@ void DoSerialise(SerialiserType &ser, SDObject *el)
       for(size_t o = 0; o < el->NumChildren(); o++)
         ser.Serialise(""_lit, el->GetChild(o));
       break;
-    case SDBasic::Array: ser.Serialise(""_lit, (rdcarray<SDObject *> &)el->data.children); break;
+    case SDBasic::Array:
+    {
+      uint64_t arraySize = el->NumChildren();
+      ser.Serialise(""_lit, arraySize);
+      // ensure all children are ready
+      for(size_t o = 0; o < el->NumChildren(); o++)
+        ser.Serialise(""_lit, el->GetChild(o));
+      break;
+    }
     case SDBasic::Null:
       // nothing to do, we serialised present flag above
       RDCASSERT(el->type.flags & SDTypeFlags::Nullable);

renderdoc/serialise/serialiser.h

@@ -630,20 +630,34 @@ public:
       }
 #endif
 
-      for(uint64_t i = 0; el && i < arrayCount; i++)
+      if(m_LazyThreshold > 0 && arrayCount > m_LazyThreshold)
       {
-        SDObject &obj = *arr.AddAndOwnChild(new SDObject("$el"_lit, TypeName<T>()));
-        m_StructureStack.push_back(&obj);
+        PushInternal();
 
-        // default to struct. This will be overwritten if appropriate
-        obj.type.basetype = SDBasic::Struct;
-        obj.type.byteSize = sizeof(T);
-        if(std::is_union<T>::value)
-          obj.type.flags |= SDTypeFlags::Union;
+        for(uint64_t i = 0; el && i < arrayCount; i++)
+          SerialiseDispatch<Serialiser, T>::Do(*this, el[i]);
 
-        SerialiseDispatch<Serialiser, T>::Do(*this, el[i]);
+        PopInternal();
 
-        m_StructureStack.pop_back();
+        arr.SetLazyArray(arrayCount, el, MakeLazySerialiser<T>());
+      }
+      else
+      {
+        for(uint64_t i = 0; el && i < arrayCount; i++)
+        {
+          SDObject &obj = *arr.AddAndOwnChild(new SDObject("$el"_lit, TypeName<T>()));
+          m_StructureStack.push_back(&obj);
+
+          // default to struct. This will be overwritten if appropriate
+          obj.type.basetype = SDBasic::Struct;
+          obj.type.byteSize = sizeof(T);
+          if(std::is_union<T>::value)
+            obj.type.flags |= SDTypeFlags::Union;
+
+          SerialiseDispatch<Serialiser, T>::Do(*this, el[i]);
+
+          m_StructureStack.pop_back();
+        }
       }
 
       m_StructureStack.pop_back();
@@ -709,18 +723,32 @@ public:
       if(IsReading())
         el.resize((int)size);
 
-      for(size_t i = 0; i < (size_t)size; i++)
+      if(m_LazyThreshold > 0 && size > m_LazyThreshold)
       {
-        SDObject &obj = *arr.AddAndOwnChild(new SDObject("$el"_lit, TypeName<U>()));
-        m_StructureStack.push_back(&obj);
+        PushInternal();
 
-        // default to struct. This will be overwritten if appropriate
-        obj.type.basetype = SDBasic::Struct;
-        obj.type.byteSize = sizeof(U);
+        for(size_t i = 0; i < (size_t)size; i++)
+          SerialiseDispatch<Serialiser, U>::Do(*this, el[i]);
 
-        SerialiseDispatch<Serialiser, U>::Do(*this, el[i]);
+        PopInternal();
 
-        m_StructureStack.pop_back();
+        arr.SetLazyArray(size, el.data(), MakeLazySerialiser<U>());
+      }
+      else
+      {
+        for(size_t i = 0; i < (size_t)size; i++)
+        {
+          SDObject &obj = *arr.AddAndOwnChild(new SDObject("$el"_lit, TypeName<U>()));
+          m_StructureStack.push_back(&obj);
+
+          // default to struct. This will be overwritten if appropriate
+          obj.type.basetype = SDBasic::Struct;
+          obj.type.byteSize = sizeof(U);
+
+          SerialiseDispatch<Serialiser, U>::Do(*this, el[i]);
+
+          m_StructureStack.pop_back();
+        }
       }
 
       m_StructureStack.pop_back();
@@ -1072,6 +1100,9 @@ public:
   // anything serialised while internal is set.
   void PushInternal() { m_InternalElement++; }
   void PopInternal() { m_InternalElement--; }
+  // this sets the current threshold for making structured data lazy for arrays above this size.
+  // If set to 0, structured data is never set as lazy
+  void SetLazyThreshold(uint32_t arraySize) { m_LazyThreshold = arraySize; }
   /////////////////////////////////////////////////////////////////////////////
 
   // for basic/leaf types. Read/written just as byte soup, MUST be plain old data
@@ -1223,6 +1254,9 @@ protected:
   Serialiser(StreamWriter *writer, Ownership own);
   Serialiser(StreamReader *reader, Ownership own, SDObject *rootStructuredObj);
 
+  template <SerialiserMode othertype>
+  friend class Serialiser;
+
   void SetDummy(bool dummy) { m_Dummy = dummy; }
 private:
   static const uint64_t ChunkAlignment = 64;
@@ -1274,6 +1308,32 @@ private:
     }
   }
 
+  template <typename T>
+  LazyGenerator MakeLazySerialiser()
+  {
+    ChunkLookup lookup = m_ChunkLookup;
+    return [lookup](const void *ptr) {
+      T &input = *(T *)ptr;
+      static StreamReader dummy(StreamReader::DummyStream);
+
+      SDObject *ret = new SDObject("$el"_lit, TypeName<T>());
+
+      ret->type.byteSize = sizeof(T);
+      if(std::is_union<T>::value)
+        ret->type.flags |= SDTypeFlags::Union;
+
+      Serialiser<SerialiserMode::Reading> ser(&dummy, Ownership::Nothing, ret);
+
+      ser.ConfigureStructuredExport(lookup, false, 0, 1.0);
+      ser.SetStreamingMode(true);
+      ser.SetDummy(true);
+
+      SerialiseDispatch<Serialiser<SerialiserMode::Reading>, T>::Do(ser, input);
+
+      return ret;
+    };
+  }
+
   void *m_pUserData = NULL;
   uint64_t m_Version = 0;
 
@@ -1295,6 +1355,7 @@ private:
   bool m_ExportStructured = false;
   bool m_ExportBuffers = false;
   int m_InternalElement = 0;
+  uint32_t m_LazyThreshold = 0;
   SDFile m_StructData;
   SDFile *m_StructuredFile = &m_StructData;
   rdcarray<SDObject *> m_StructureStack;

renderdoc/serialise/serialiser_tests.cpp

@@ -255,13 +255,13 @@ TEST_CASE("Read/write via structured of basic types", "[serialiser]")
     CHECK(chunk.type.name == "Chunk");
     CHECK(chunk.NumChildren() == 16);
 
-    for(SDObject *o : chunk)
+    for(const SDObject *o : chunk)
       REQUIRE(o);
 
     int childIdx = 0;
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "a");
       CHECK(o.type.name == "int64_t");
@@ -273,7 +273,7 @@ TEST_CASE("Read/write via structured of basic types", "[serialiser]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "b");
       CHECK(o.type.name == "uint64_t");
@@ -285,7 +285,7 @@ TEST_CASE("Read/write via structured of basic types", "[serialiser]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "c");
       CHECK(o.type.name == "int32_t");
@@ -297,7 +297,7 @@ TEST_CASE("Read/write via structured of basic types", "[serialiser]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "d");
       CHECK(o.type.name == "uint32_t");
@@ -309,7 +309,7 @@ TEST_CASE("Read/write via structured of basic types", "[serialiser]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "e");
       CHECK(o.type.name == "int16_t");
@@ -321,7 +321,7 @@ TEST_CASE("Read/write via structured of basic types", "[serialiser]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "f");
       CHECK(o.type.name == "uint16_t");
@@ -333,7 +333,7 @@ TEST_CASE("Read/write via structured of basic types", "[serialiser]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "g");
       CHECK(o.type.name == "int8_t");
@@ -345,7 +345,7 @@ TEST_CASE("Read/write via structured of basic types", "[serialiser]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "h");
       CHECK(o.type.name == "uint8_t");
@@ -357,7 +357,7 @@ TEST_CASE("Read/write via structured of basic types", "[serialiser]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "i");
       CHECK(o.type.name == "bool");
@@ -369,7 +369,7 @@ TEST_CASE("Read/write via structured of basic types", "[serialiser]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "j");
       CHECK(o.type.name == "char");
@@ -381,7 +381,7 @@ TEST_CASE("Read/write via structured of basic types", "[serialiser]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "k");
       CHECK(o.type.name == "double");
@@ -393,7 +393,7 @@ TEST_CASE("Read/write via structured of basic types", "[serialiser]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "l");
       CHECK(o.type.name == "float");
@@ -405,7 +405,7 @@ TEST_CASE("Read/write via structured of basic types", "[serialiser]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "m");
       CHECK(o.type.name == "string");
@@ -417,7 +417,7 @@ TEST_CASE("Read/write via structured of basic types", "[serialiser]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "n");
       CHECK(o.type.name == "string");
@@ -429,7 +429,7 @@ TEST_CASE("Read/write via structured of basic types", "[serialiser]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "s");
       CHECK(o.type.name == "string");
@@ -441,7 +441,7 @@ TEST_CASE("Read/write via structured of basic types", "[serialiser]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "t");
       CHECK(o.type.name == "int32_t");
@@ -935,13 +935,13 @@ TEST_CASE("Read/write container types", "[serialiser][structured]")
 
     CHECK(chunk.NumChildren() == 2);
 
-    for(SDObject *o : chunk)
+    for(const SDObject *o : chunk)
       REQUIRE(o);
 
     int childIdx = 0;
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "v");
       CHECK(o.type.basetype == SDBasic::Array);
@@ -949,7 +949,7 @@ TEST_CASE("Read/write container types", "[serialiser][structured]")
       CHECK(o.type.flags == SDTypeFlags::NoFlags);
       CHECK(o.NumChildren() == 6);
 
-      for(SDObject *child : o)
+      for(const SDObject *child : o)
       {
         CHECK(child->type.basetype == SDBasic::SignedInteger);
         CHECK(child->type.byteSize == 4);
@@ -964,7 +964,7 @@ TEST_CASE("Read/write container types", "[serialiser][structured]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "p");
       CHECK(o.type.name == "pair");
@@ -974,7 +974,7 @@ TEST_CASE("Read/write container types", "[serialiser][structured]")
       CHECK(o.NumChildren() == 2);
 
       {
-        SDObject &first = *o.GetChild(0);
+        const SDObject &first = *o.GetChild(0);
 
         CHECK(first.name == "first");
         CHECK(first.type.name == "float");
@@ -986,7 +986,7 @@ TEST_CASE("Read/write container types", "[serialiser][structured]")
       }
 
       {
-        SDObject &second = *o.GetChild(1);
+        const SDObject &second = *o.GetChild(1);
 
         CHECK(second.name == "second");
         CHECK(second.type.name == "string");
@@ -1227,13 +1227,13 @@ TEST_CASE("Read/write complex types", "[serialiser][structured]")
 
     CHECK(chunk.NumChildren() == 5);
 
-    for(SDObject *o : chunk)
+    for(const SDObject *o : chunk)
       REQUIRE(o);
 
     int childIdx = 0;
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "enumVal");
       CHECK(o.type.basetype == SDBasic::Enum);
@@ -1244,7 +1244,7 @@ TEST_CASE("Read/write complex types", "[serialiser][structured]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "sparseStructArray");
       CHECK(o.type.basetype == SDBasic::Array);
@@ -1252,7 +1252,7 @@ TEST_CASE("Read/write complex types", "[serialiser][structured]")
       CHECK(o.type.flags == SDTypeFlags::NoFlags);
       CHECK(o.NumChildren() == 10);
 
-      for(SDObject *child : o)
+      for(const SDObject *child : o)
       {
         CHECK(child->type.basetype == SDBasic::Struct);
         CHECK(child->type.name == "struct1");
@@ -1289,7 +1289,7 @@ TEST_CASE("Read/write complex types", "[serialiser][structured]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "complex");
       CHECK(o.type.name == "struct2");
@@ -1299,7 +1299,7 @@ TEST_CASE("Read/write complex types", "[serialiser][structured]")
       CHECK(o.NumChildren() == 3);
 
       {
-        SDObject &c = *o.GetChild(0);
+        const SDObject &c = *o.GetChild(0);
 
         CHECK(c.name == "name");
         CHECK(c.type.name == "string");
@@ -1310,13 +1310,13 @@ TEST_CASE("Read/write complex types", "[serialiser][structured]")
       }
 
       {
-        SDObject &c = *o.GetChild(1);
+        const SDObject &c = *o.GetChild(1);
 
         CHECK(c.name == "floats");
         CHECK(c.type.basetype == SDBasic::Array);
         CHECK(c.type.flags == SDTypeFlags::NoFlags);
         CHECK(c.NumChildren() == 3);
-        for(SDObject *ch : c)
+        for(const SDObject *ch : c)
         {
           CHECK(ch->type.basetype == SDBasic::Float);
           CHECK(ch->type.byteSize == 4);
@@ -1328,13 +1328,13 @@ TEST_CASE("Read/write complex types", "[serialiser][structured]")
       }
 
       {
-        SDObject &c = *o.GetChild(2);
+        const SDObject &c = *o.GetChild(2);
 
         CHECK(c.name == "viewports");
         CHECK(c.type.basetype == SDBasic::Array);
         CHECK(c.type.flags == SDTypeFlags::NoFlags);
         CHECK(c.NumChildren() == 4);
-        for(SDObject *ch : c)
+        for(const SDObject *ch : c)
         {
           CHECK(ch->type.basetype == SDBasic::Struct);
           CHECK(ch->type.name == "struct1");
@@ -1348,7 +1348,7 @@ TEST_CASE("Read/write complex types", "[serialiser][structured]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "inputParam1");
       CHECK(o.type.basetype == SDBasic::Struct);
@@ -1361,7 +1361,7 @@ TEST_CASE("Read/write complex types", "[serialiser][structured]")
     }
 
     {
-      SDObject &o = *chunk.GetChild(childIdx++);
+      const SDObject &o = *chunk.GetChild(childIdx++);
 
       CHECK(o.name == "inputParam2");
       CHECK(o.type.basetype == SDBasic::Null);