diff --git a/qrenderdoc/Code/pyrenderdoc/renderdoc.i b/qrenderdoc/Code/pyrenderdoc/renderdoc.i index 03278bc7d..4e9ea10ee 100644 --- a/qrenderdoc/Code/pyrenderdoc/renderdoc.i +++ b/qrenderdoc/Code/pyrenderdoc/renderdoc.i @@ -12,6 +12,7 @@ // swig's pre-processor trips up on this definition, and it's only needed for template expansions // in the conversion code #define DECLARE_REFLECTION_STRUCT(t) +#define DECLARE_REFLECTION_ENUM(t) // use documentation for docstrings #define DOCUMENT(text) %feature("docstring") text diff --git a/renderdoc/CMakeLists.txt b/renderdoc/CMakeLists.txt index 841edde2a..bed4db459 100644 --- a/renderdoc/CMakeLists.txt +++ b/renderdoc/CMakeLists.txt @@ -134,6 +134,7 @@ set(sources serialise/rdcfile.h serialise/codecs/xml_codec.cpp serialise/comp_io_tests.cpp + serialise/serialiser_tests.cpp serialise/streamio_tests.cpp strings/grisu2.cpp strings/string_utils.cpp diff --git a/renderdoc/api/replay/renderdoc_replay.h b/renderdoc/api/replay/renderdoc_replay.h index d6fd431af..4f935c270 100644 --- a/renderdoc/api/replay/renderdoc_replay.h +++ b/renderdoc/api/replay/renderdoc_replay.h @@ -336,15 +336,6 @@ inline enum_name operator++(enum_name &a) \ #include "basic_types.h" #include "stringise.h" - -#ifndef DECLARE_REFLECTION_STRUCT -#define DECLARE_REFLECTION_STRUCT(type) DECLARE_STRINGISE_TYPE(type) -#endif - -#ifndef DECLARE_REFLECTION_ENUM -#define DECLARE_REFLECTION_ENUM(type) DECLARE_STRINGISE_TYPE(type) -#endif - #include "structured_data.h" #ifdef RENDERDOC_EXPORTS diff --git a/renderdoc/api/replay/stringise.h b/renderdoc/api/replay/stringise.h index 78ea964fe..0b16d0294 100644 --- a/renderdoc/api/replay/stringise.h +++ b/renderdoc/api/replay/stringise.h @@ -153,3 +153,31 @@ inline const char *TypeName(); { \ return #type; \ } + +// This is a little bit ugly, but not *too* much. We declare the macro for serialised types to +// forward-declare the template used by the serialiser. This will be visible externally, but it will +// do nothing as the template won't be invoked externally. It means we can use the correct macro in +// the external interface headers to pre-declare them as serialisable, without having to have the +// public interface include serialiser.h which leads to horrible circular dependencies. + +// main serialisation templated function that is specialised for any type +// that needs to be serialised. Implementations will either be a struct type, +// in which case they'll call Serialise() on each of their members, or +// they'll be some custom leaf type (most basic types are specialised in this +// header) which will call SerialiseValue(). + +template +void DoSerialise(SerialiserType &ser, T &el); + +#ifndef DECLARE_REFLECTION_STRUCT +#define DECLARE_REFLECTION_STRUCT(type) \ + DECLARE_STRINGISE_TYPE(type) \ + template \ + void DoSerialise(SerialiserType &ser, type &el); +#endif + +// enums don't have anything special to do to serialise, they're all handled automagically once they +// have a DoStringise function +#ifndef DECLARE_REFLECTION_ENUM +#define DECLARE_REFLECTION_ENUM(type) DECLARE_STRINGISE_TYPE(type) +#endif diff --git a/renderdoc/common/globalconfig.h b/renderdoc/common/globalconfig.h index a1c24c3ef..bbde858ca 100644 --- a/renderdoc/common/globalconfig.h +++ b/renderdoc/common/globalconfig.h @@ -97,6 +97,13 @@ #endif +// is size_t a real separate type, not just typedef'd to uint32_t or uint64_t (or equivalent)? +#if defined(RENDERDOC_PLATFORM_APPLE) +#define RDOC_SIZET_SEP_TYPE OPTION_ON +#else +#define RDOC_SIZET_SEP_TYPE OPTION_OFF +#endif + #if defined(RENDERDOC_WINDOWING_XLIB) #define RDOC_XLIB OPTION_ON #else diff --git a/renderdoc/renderdoc.vcxproj b/renderdoc/renderdoc.vcxproj index 3a1e7a96c..6f502c6c2 100644 --- a/renderdoc/renderdoc.vcxproj +++ b/renderdoc/renderdoc.vcxproj @@ -402,6 +402,7 @@ + diff --git a/renderdoc/renderdoc.vcxproj.filters b/renderdoc/renderdoc.vcxproj.filters index 4ef2d4564..e11f2a167 100644 --- a/renderdoc/renderdoc.vcxproj.filters +++ b/renderdoc/renderdoc.vcxproj.filters @@ -632,6 +632,9 @@ Common\Serialise\Codecs + + Common\Serialise + diff --git a/renderdoc/serialise/serialiser.cpp b/renderdoc/serialise/serialiser.cpp index 4c1bb793b..80a756075 100644 --- a/renderdoc/serialise/serialiser.cpp +++ b/renderdoc/serialise/serialiser.cpp @@ -1,8 +1,7 @@ /****************************************************************************** * The MIT License (MIT) * - * Copyright (c) 2015-2017 Baldur Karlsson - * Copyright (c) 2014 Crytek + * Copyright (c) 2017 Baldur Karlsson * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -23,20 +22,14 @@ * THE SOFTWARE. ******************************************************************************/ +// used to avoid instantiating templates too early in the header +#define SERIALISER_IMPL + #include "serialiser.h" -#include -#include "3rdparty/lz4/lz4.h" -#include "common/timing.h" #include "core/core.h" #include "strings/string_utils.h" -#if ENABLED(RDOC_MSVS) -// warning C4422: 'snprintf' : too many arguments passed for format string -// false positive as VS is trying to parse renderdoc's custom format strings -#pragma warning(disable : 4422) -#endif - -#if ENABLED(RDOC_DEVEL) +#if !defined(RELEASE) int64_t Chunk::m_LiveChunks = 0; int64_t Chunk::m_TotalMem = 0; @@ -44,1892 +37,696 @@ int64_t Chunk::m_MaxChunks = 0; #endif -const uint32_t Serialiser::MAGIC_HEADER = MAKE_FOURCC('R', 'D', 'O', 'C'); -const uint64_t Serialiser::BufferAlignment = 64; +///////////////////////////////////////////////////////////// +// Read Serialiser functions -// based on blockStreaming_doubleBuffer.c in lz4 examples -struct CompressedFileIO +template <> +Serialiser::Serialiser(StreamReader *reader, Ownership own) { - // large block size - static const size_t BlockSize = 64 * 1024; + m_Read = reader; + m_Write = NULL; - CompressedFileIO(FILE *f) - { - m_F = f; - LZ4_resetStream(&m_LZ4Comp); - LZ4_setStreamDecode(&m_LZ4Decomp, NULL, 0); - m_CompressedSize = m_UncompressedSize = 0; - m_PageIdx = m_PageOffset = 0; - m_PageData = 0; - - m_CompressSize = LZ4_COMPRESSBOUND(BlockSize); - m_CompressBuf = new byte[m_CompressSize]; - } - - ~CompressedFileIO() { SAFE_DELETE_ARRAY(m_CompressBuf); } - uint64_t GetCompressedSize() { return m_CompressedSize; } - uint64_t GetUncompressedSize() { return m_UncompressedSize; } - // write out some data - accumulate into the input pages, then - // when a page is full call Flush() to flush it out to disk - void Write(const void *data, size_t len) - { - if(data == NULL || len == 0) - return; - - m_UncompressedSize += (uint64_t)len; - - const byte *src = (const byte *)data; - - size_t remainder = 0; - - // loop continually, writing up to BlockSize out of what remains of data - do - { - remainder = 0; - - // if we're about to copy more than the page, copy only - // what will fit, then copy the remainder after flushing - if(m_PageOffset + len > BlockSize) - { - remainder = len - (BlockSize - m_PageOffset); - len = BlockSize - m_PageOffset; - } - - memcpy(m_InPages[m_PageIdx] + m_PageOffset, src, len); - m_PageOffset += len; - - if(remainder > 0) - { - Flush(); // this will swap the input pages and reset the page offset - - src += len; - len = remainder; - } - } while(remainder > 0); - } - - // flush out the current page to disk - void Flush() - { - // m_PageOffset is the amount written, usually equal to BlockSize except the last block. - int32_t compSize = LZ4_compress_fast_continue(&m_LZ4Comp, (const char *)m_InPages[m_PageIdx], - (char *)m_CompressBuf, (int)m_PageOffset, - (int)m_CompressSize, 1); - - if(compSize < 0) - { - RDCERR("Error compressing: %i", compSize); - return; - } - - FileIO::fwrite(&compSize, sizeof(compSize), 1, m_F); - FileIO::fwrite(m_CompressBuf, 1, compSize, m_F); - - m_CompressedSize += uint64_t(compSize) + sizeof(int32_t); - - m_PageOffset = 0; - m_PageIdx = 1 - m_PageIdx; - } - - // Reset back to 0, only makes sense when reading as writing can't be undone - void Reset() - { - LZ4_setStreamDecode(&m_LZ4Decomp, NULL, 0); - m_CompressedSize = m_UncompressedSize = 0; - m_PageIdx = 0; - m_PageOffset = 0; - } - - // read out some data - if the input page is empty we fill - // the next page with data from disk - void Read(byte *data, size_t len) - { - if(data == NULL || len == 0) - return; - - m_UncompressedSize += (uint64_t)len; - - // loop continually, writing up to BlockSize out of what remains of data - do - { - size_t readamount = len; - - // if we're about to copy more than the page, copy only - // what will fit, then copy the remainder after refilling - if(readamount > m_PageData) - readamount = m_PageData; - - if(readamount > 0) - { - memcpy(data, m_InPages[m_PageIdx] + m_PageOffset, readamount); - - m_PageOffset += readamount; - m_PageData -= readamount; - - data += readamount; - len -= readamount; - } - - if(len > 0) - FillBuffer(); // this will swap the input pages and reset the page offset - } while(len > 0); - } - - void FillBuffer() - { - int32_t compSize = 0; - - FileIO::fread(&compSize, sizeof(compSize), 1, m_F); - size_t numRead = FileIO::fread(m_CompressBuf, 1, compSize, m_F); - - m_CompressedSize += uint64_t(compSize); - - m_PageIdx = 1 - m_PageIdx; - - int32_t decompSize = LZ4_decompress_safe_continue( - &m_LZ4Decomp, (const char *)m_CompressBuf, (char *)m_InPages[m_PageIdx], compSize, BlockSize); - - if(decompSize < 0) - { - RDCERR("Error decompressing: %i (%i / %i)", decompSize, int(numRead), compSize); - return; - } - - m_PageOffset = 0; - m_PageData = decompSize; - } - - static void Decompress(byte *destBuf, const byte *srcBuf, size_t len) - { - LZ4_streamDecode_t lz4; - LZ4_setStreamDecode(&lz4, NULL, 0); - - const byte *srcBufEnd = srcBuf + len; - - while(srcBuf + 4 < srcBufEnd) - { - const int32_t *compSize = (const int32_t *)srcBuf; - srcBuf = (const byte *)(compSize + 1); - - if(srcBuf + *compSize > srcBufEnd) - break; - - int32_t decompSize = LZ4_decompress_safe_continue(&lz4, (const char *)srcBuf, (char *)destBuf, - *compSize, BlockSize); - - if(decompSize < 0) - return; - - srcBuf += *compSize; - destBuf += decompSize; - } - } - - LZ4_stream_t m_LZ4Comp; - LZ4_streamDecode_t m_LZ4Decomp; - FILE *m_F; - uint64_t m_CompressedSize, m_UncompressedSize; - - byte m_InPages[2][BlockSize]; - size_t m_PageIdx, m_PageOffset, m_PageData; - - byte *m_CompressBuf; - size_t m_CompressSize; -}; - -Chunk::Chunk(Serialiser *ser, uint32_t chunkType, bool temporary) -{ - m_Length = (uint32_t)ser->GetOffset(); - - RDCASSERT(ser->GetOffset() < 0xffffffff); - - m_ChunkType = chunkType; - - m_Temporary = temporary; - - if(ser->HasAlignedData()) - { - m_Data = AllocAlignedBuffer(m_Length); - m_AlignedData = true; - } - else - { - m_Data = new byte[m_Length]; - m_AlignedData = false; - } - - memcpy(m_Data, ser->GetRawPtr(0), m_Length); - - if(ser->GetDebugText()) - m_DebugStr = ser->GetDebugStr(); - - ser->Rewind(); - -#if ENABLED(RDOC_DEVEL) - int64_t newval = Atomic::Inc64(&m_LiveChunks); - Atomic::ExchAdd64(&m_TotalMem, m_Length); - - if(newval > m_MaxChunks) - { - int breakpointme = 0; - (void)breakpointme; - } - - m_MaxChunks = RDCMAX(newval, m_MaxChunks); -#endif + m_Ownership = own; } -Chunk *Chunk::Duplicate() +template <> +Serialiser::~Serialiser() { - Chunk *ret = new Chunk(); - ret->m_DebugStr = m_DebugStr; - ret->m_Length = m_Length; - ret->m_ChunkType = m_ChunkType; - ret->m_Temporary = m_Temporary; - ret->m_AlignedData = m_AlignedData; - - if(m_AlignedData) - ret->m_Data = AllocAlignedBuffer(m_Length); - else - ret->m_Data = new byte[m_Length]; - - memcpy(ret->m_Data, m_Data, m_Length); - -#if ENABLED(RDOC_DEVEL) - int64_t newval = Atomic::Inc64(&m_LiveChunks); - Atomic::ExchAdd64(&m_TotalMem, m_Length); - - if(newval > m_MaxChunks) - { - int breakpointme = 0; - (void)breakpointme; - } - - m_MaxChunks = RDCMAX(newval, m_MaxChunks); -#endif - - return ret; + if(m_Ownership == Ownership::Stream && m_Read) + delete m_Read; } -Chunk::~Chunk() +template <> +uint32_t Serialiser::BeginChunk(uint32_t, uint32_t) { -#if ENABLED(RDOC_DEVEL) - Atomic::Dec64(&m_LiveChunks); - Atomic::ExchAdd64(&m_TotalMem, -int64_t(m_Length)); -#endif + uint32_t chunkID = 0; - if(m_AlignedData) - { - if(m_Data) - FreeAlignedBuffer(m_Data); + m_ChunkMetadata = SDChunkMetaData(); - m_Data = NULL; - } - else { - SAFE_DELETE_ARRAY(m_Data); + uint32_t c = 0; + bool success = m_Read->Read(c); + + // Chunk index 0 is not allowed in normal situations, and allows us to indicate some control + // bytes. Currently this is unused + RDCASSERT(c != 0 || !success); + + chunkID = c & ChunkIndexMask; + + ///////////////// + + m_ChunkMetadata.chunkID = chunkID; + + if(c & ChunkCallstack) + { + uint32_t numFrames = 0; + m_Read->Read(numFrames); + + m_ChunkMetadata.callstack.resize((size_t)numFrames); + m_Read->Read(m_ChunkMetadata.callstack.data(), m_ChunkMetadata.callstack.byteSize()); + } + + if(c & ChunkThreadID) + m_Read->Read(m_ChunkMetadata.threadID); + + if(c & ChunkDuration) + m_Read->Read(m_ChunkMetadata.durationMicro); + + if(c & ChunkTimestamp) + m_Read->Read(m_ChunkMetadata.timestampMicro); + + m_Read->Read(m_ChunkMetadata.length); + + m_LastChunkOffset = m_Read->GetOffset(); } + + if(ExportStructure()) + { + std::string name = m_ChunkLookup ? m_ChunkLookup(chunkID) : ""; + + if(name.empty()) + name = ""; + + SDChunk *chunk = new SDChunk(name.c_str()); + chunk->metadata = m_ChunkMetadata; + + m_StructuredFile->chunks.push_back(chunk); + m_StructureStack.push_back(chunk); + + m_InternalElement = false; + } + + return chunkID; } -/* - - ----------------------------- - File format for version 0x31: - - uint64_t MAGIC_HEADER; - uint64_t version = 0x00000031; - uint64_t filesize; - uint64_t resolveDBSize; - - if(resolveDBSize > 0) - { - byte resolveDB[resolveDBSize]; - byte paddingTo16Alignment[]; - } - - byte captureData[]; // remainder of the file - - ----------------------------- - File format for version 0x32: - - uint64_t MAGIC_HEADER; - uint64_t version = 0x00000032; - - 1 or more sections: - - Section - { - char isASCII = '\0' or 'A'; // indicates the section is ASCII or binary. ASCII allows for easy - appending by hand/script - if(isASCII == 'A') - { - // ASCII sections are discouraged for tools, but useful for hand-editing by just - // appending a simple text file - char newline = '\n'; - char length[]; // length of just section data below, as decimal string - char newline = '\n'; - char sectionType[]; // section type, see SectionType enum, as decimal string. - char newline = '\n'; - char sectionName[]; // UTF-8 string name of section. - char newline = '\n'; - // sectionName is an arbitrary string. If two sections exist with the - // exact same name, the last one to occur in the file will be taken, the - // others ignored. - - byte sectiondata[ interpret(length) ]; // section data - } - else if(isASCII == '\0') - { - byte zero[3]; // pad out the above character with 0 bytes. Reserved for future use - uint32_t sectionFlags; // section flags - e.g. is compressed or not. - uint32_t sectionType; // section type enum, see SectionType. Could be eSectionType_Unknown - uint32_t sectionLength; // byte length of the actual section data - uint32_t sectionNameLength; // byte length of the string below (minimum 1, for null terminator) - char sectionName[sectionNameLength]; // UTF-8 string name of section, optional. - - byte sectiondata[length]; // actual contents of the section - - // note: compressed sections will contain the uncompressed length as a uint64_t - // before the compressed data. - } - }; - - // remainder of the file is tightly packed/unaligned section structures. - // The first section must always be the actual frame capture data in - // binary form - Section sections[]; - -*/ - -struct FileHeader +template <> +void Serialiser::SkipCurrentChunk() { - FileHeader() + if(ExportStructure()) { - magic = Serialiser::MAGIC_HEADER; - version = Serialiser::SERIALISE_VERSION; - } + RDCASSERTMSG("Skipping chunk after we've begun serialising!", m_StructureStack.size() == 1, + m_StructureStack.size()); - uint64_t magic; - uint64_t version; -}; + SDObject ¤t = *m_StructureStack.back(); -struct BinarySectionHeader -{ - byte isASCII; // 0x0 - byte zero[3]; // 0x0, 0x0, 0x0 - Serialiser::SectionFlags sectionFlags; // section flags - e.g. is compressed or not. - Serialiser::SectionType - sectionType; // section type enum, see SectionType. Could be eSectionType_Unknown - uint32_t sectionLength; // byte length of the actual section data - uint32_t sectionNameLength; // byte length of the string below (could be 0) - char name[1]; // actually sectionNameLength, but at least 1 for null terminator + current.data.basic.numChildren++; + current.data.children.push_back(new SDObject("Opaque chunk", "Byte Buffer")); - // char name[sectionNameLength]; - // byte data[sectionLength]; -}; + SDObject &obj = *current.data.children.back(); + obj.type.basetype = SDBasic::Buffer; + obj.type.byteSize = m_ChunkMetadata.length; -#define RETURNCORRUPT(...) \ - { \ - RDCERR(__VA_ARGS__); \ - m_ErrorCode = eSerError_Corrupt; \ - m_HasError = true; \ - return; \ - } - -Serialiser::Serialiser(size_t length, const byte *memoryBuf, bool fileheader) - : m_pCallstack(NULL), m_pResolver(NULL), m_Buffer(NULL) -{ - m_ResolverThread = 0; - - // ensure binary sizes of enums - RDCCOMPILE_ASSERT(sizeof(SectionFlags) == sizeof(uint32_t), "Section flags not in uint32"); - RDCCOMPILE_ASSERT(sizeof(SectionType) == sizeof(uint32_t), "Section type not in uint32"); - - RDCCOMPILE_ASSERT(offsetof(BinarySectionHeader, name) == sizeof(uint32_t) * 5, - "BinarySectionHeader size has changed or contains padding"); - - Reset(); - - m_Mode = READING; - m_DebugEnabled = false; - - m_FileSize = 0; - - if(!fileheader) - { - m_BufferSize = length; - m_CurrentBufferSize = (size_t)m_BufferSize; - m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); - - m_SerVer = SERIALISE_VERSION; - - memcpy(m_Buffer, memoryBuf, m_CurrentBufferSize); - return; - } - - FileHeader *header = (FileHeader *)memoryBuf; - - if(length < sizeof(FileHeader)) - { - RDCERR("Can't read from in-memory buffer, truncated header"); - m_ErrorCode = eSerError_Corrupt; - m_HasError = true; - return; - } - - if(header->magic != MAGIC_HEADER) - { - char magicRef[5] = {0}; - char magicFile[5] = {0}; - memcpy(magicRef, &MAGIC_HEADER, sizeof(uint32_t)); - memcpy(magicFile, &header->magic, sizeof(uint32_t)); - RDCWARN("Invalid in-memory buffer. Expected magic %s, got %s", magicRef, magicFile); - - m_ErrorCode = eSerError_Corrupt; - m_HasError = true; - return; - } - - const byte *memoryBufEnd = memoryBuf + length; - - m_SerVer = header->version; - - if(header->version == 0x00000031) // backwards compatibility - { - memoryBuf += sizeof(FileHeader); - - if(length < sizeof(FileHeader) + sizeof(uint64_t) * 2) + if(m_StructureStack.size() == 1) { - RDCERR("Can't read from in-memory buffer, truncated header"); - m_ErrorCode = eSerError_Corrupt; - m_HasError = true; + SDChunk *chunk = (SDChunk *)m_StructureStack.back(); + chunk->metadata.flags |= SDChunkFlags::OpaqueChunk; + } + } + + { + uint64_t readBytes = m_Read->GetOffset() - m_LastChunkOffset; + + if(readBytes > m_ChunkMetadata.length) + { + RDCERR("Can't skip current chunk outside of {BeginChunk, EndChunk}"); return; } - uint64_t *fileSize = (uint64_t *)memoryBuf; - memoryBuf += sizeof(uint64_t); - - uint64_t *resolveDBSize = (uint64_t *)memoryBuf; - memoryBuf += sizeof(uint64_t); - - if(*fileSize < length) + if(readBytes > 0) { - RDCERR("Overlong in-memory buffer. Expected length 0x016llx, got 0x016llx", *fileSize, length); - - m_ErrorCode = eSerError_Corrupt; - m_HasError = true; - return; + RDCWARN("Partially consumed bytes at SkipCurrentChunk - blob data will be truncated"); } - // for in-memory case we don't need to load up the resolve db + uint64_t chunkBytes = m_ChunkMetadata.length - readBytes; - Section *frameCap = new Section(); - frameCap->type = eSectionType_FrameCapture; - frameCap->flags = eSectionFlag_None; - frameCap->fileoffset = 0; // irrelevant - frameCap->name = "renderdoc/internal/framecapture"; - frameCap->size = uint64_t(memoryBufEnd - memoryBuf); - - memoryBuf = AlignUpPtr(memoryBuf + *resolveDBSize, 16); - - m_Sections.push_back(frameCap); - m_KnownSections[eSectionType_FrameCapture] = frameCap; - } - else if(header->version == SERIALISE_VERSION) - { - memoryBuf += sizeof(FileHeader); - - // when loading in-memory we only care about the first section, which should be binary - const BinarySectionHeader *sectionHeader = (const BinarySectionHeader *)memoryBuf; - - // verify validity - if(memoryBuf + offsetof(BinarySectionHeader, name) >= memoryBufEnd) + if(ExportStructure() && m_ExportBuffers) { - RDCERR("Truncated binary section header"); + SDObject ¤t = *m_StructureStack.back(); - m_ErrorCode = eSerError_Corrupt; - m_HasError = true; - return; - } + SDObject &obj = *current.data.children.back(); - if(sectionHeader->isASCII != 0 || sectionHeader->zero[0] != 0 || sectionHeader->zero[1] != 0 || - sectionHeader->zero[2] != 0) - { - RDCERR("Unexpected non-binary section first in capture when loading in-memory"); + obj.data.basic.u = m_StructuredFile->buffers.size(); - m_ErrorCode = eSerError_Corrupt; - m_HasError = true; - return; - } + bytebuf *alloc = new bytebuf; + alloc->resize((size_t)chunkBytes); + m_Read->Read(alloc->data(), (size_t)chunkBytes); - if(sectionHeader->sectionType != eSectionType_FrameCapture) - { - RDCERR("Expected first section to be frame capture, got type %x", sectionHeader->sectionType); - - m_ErrorCode = eSerError_Corrupt; - m_HasError = true; - return; - } - - memoryBuf += offsetof(BinarySectionHeader, name); - memoryBuf += sectionHeader->sectionNameLength; // skip name - - if(memoryBuf >= memoryBufEnd) - { - RDCERR("Truncated binary section header"); - - m_ErrorCode = eSerError_Corrupt; - m_HasError = true; - return; - } - - Section *frameCap = new Section(); - frameCap->fileoffset = 0; // irrelevant - frameCap->data.assign(memoryBuf, memoryBufEnd); - frameCap->name = sectionHeader->name; - frameCap->type = sectionHeader->sectionType; - frameCap->flags = sectionHeader->sectionFlags; - - uint64_t *uncompLength = (uint64_t *)memoryBuf; - - memoryBuf += sizeof(uint64_t); - - if(memoryBuf >= memoryBufEnd) - { - RDCERR("Truncated binary section header"); - - m_ErrorCode = eSerError_Corrupt; - m_HasError = true; - - SAFE_DELETE(frameCap); - return; - } - - frameCap->size = *uncompLength; - - m_KnownSections[eSectionType_FrameCapture] = frameCap; - m_Sections.push_back(frameCap); - } - else - { - RDCERR( - "Capture file from wrong version. This program is on logfile version %llu, file is logfile " - "version %llu", - SERIALISE_VERSION, header->version); - - m_ErrorCode = eSerError_UnsupportedVersion; - m_HasError = true; - return; - } - - m_BufferSize = m_KnownSections[eSectionType_FrameCapture]->size; - m_CurrentBufferSize = (size_t)m_BufferSize; - m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); - - if(m_KnownSections[eSectionType_FrameCapture]->flags & eSectionFlag_LZ4Compressed) - { - CompressedFileIO::Decompress(m_Buffer, memoryBuf, memoryBufEnd - memoryBuf); - } - else - { - memcpy(m_Buffer, memoryBuf, m_CurrentBufferSize); - } -} - -Serialiser::Serialiser(const char *path, Mode mode, bool debugMode, uint64_t sizeHint) - : m_pCallstack(NULL), m_pResolver(NULL), m_Buffer(NULL) -{ - m_ResolverThread = 0; - - Reset(); - - m_Filename = path ? path : ""; - - m_Mode = mode; - m_DebugEnabled = debugMode; - - m_FileSize = 0; - - FileHeader header; - - if(mode == READING) - { - m_ReadFileHandle = FileIO::fopen(m_Filename.c_str(), "rb"); - - if(!m_ReadFileHandle) - { - RDCERR("Can't open capture file '%s' for read - errno %d", m_Filename.c_str(), errno); - m_ErrorCode = eSerError_FileIO; - m_HasError = true; - return; - } - - FileIO::fseek64(m_ReadFileHandle, 0, SEEK_END); - - m_FileSize = FileIO::ftell64(m_ReadFileHandle); - - FileIO::fseek64(m_ReadFileHandle, 0, SEEK_SET); - - RDCDEBUG("Opened capture file for read"); - - FileIO::fread(&header, 1, sizeof(FileHeader), m_ReadFileHandle); - - if(header.magic != MAGIC_HEADER) - { - RDCWARN("Invalid capture file. Expected magic %08x, got %08x", MAGIC_HEADER, - (uint32_t)header.magic); - - m_ErrorCode = eSerError_Corrupt; - m_HasError = true; - FileIO::fclose(m_ReadFileHandle); - m_ReadFileHandle = 0; - return; - } - - m_SerVer = header.version; - - if(header.version == 0x00000031) // backwards compatibility - { - uint64_t headerRemainder[2]; - - FileIO::fread(&headerRemainder, 1, sizeof(headerRemainder), m_ReadFileHandle); - - FileIO::fseek64(m_ReadFileHandle, 0, SEEK_END); - - uint64_t realLength = FileIO::ftell64(m_ReadFileHandle); - if(headerRemainder[0] != realLength) - { - RDCERR("Truncated/overlong capture file. Expected length 0x016llx, got 0x016llx", - headerRemainder[0], realLength); - - m_ErrorCode = eSerError_Corrupt; - m_HasError = true; - FileIO::fclose(m_ReadFileHandle); - m_ReadFileHandle = 0; - return; - } - - FileIO::fseek64(m_ReadFileHandle, sizeof(FileHeader) + sizeof(headerRemainder), SEEK_SET); - - size_t resolveDBSize = (size_t)headerRemainder[1]; - - if(resolveDBSize > 0) - { - Section *resolveDB = new Section(); - resolveDB->type = eSectionType_ResolveDatabase; - resolveDB->flags = eSectionFlag_None; - resolveDB->fileoffset = sizeof(FileHeader) + sizeof(headerRemainder); - resolveDB->name = "renderdoc/internal/resolvedb"; - resolveDB->data.resize(resolveDBSize); - - // read resolve DB entirely into memory - FileIO::fread(&resolveDB->data[0], 1, resolveDBSize, m_ReadFileHandle); - - m_Sections.push_back(resolveDB); - m_KnownSections[eSectionType_ResolveDatabase] = resolveDB; - } - - // seek to frame capture data - FileIO::fseek64(m_ReadFileHandle, - AlignUp16(sizeof(FileHeader) + sizeof(headerRemainder) + resolveDBSize), - SEEK_SET); - - Section *frameCap = new Section(); - frameCap->type = eSectionType_FrameCapture; - frameCap->flags = eSectionFlag_None; - frameCap->fileoffset = FileIO::ftell64(m_ReadFileHandle); - frameCap->name = "renderdoc/internal/framecapture"; - frameCap->size = realLength - frameCap->fileoffset; - - m_Sections.push_back(frameCap); - m_KnownSections[eSectionType_FrameCapture] = frameCap; - } - else if(header.version == SERIALISE_VERSION) - { - while(!FileIO::feof(m_ReadFileHandle)) - { - BinarySectionHeader sectionHeader = {0}; - byte *reading = (byte *)§ionHeader; - - FileIO::fread(reading, 1, 1, m_ReadFileHandle); - reading++; - - if(FileIO::feof(m_ReadFileHandle)) - break; - - if(sectionHeader.isASCII == 'A') - { - // ASCII section - char c = 0; - FileIO::fread(&c, 1, 1, m_ReadFileHandle); - if(c != '\n') - RETURNCORRUPT("Invalid ASCII data section '%hhx'", c); - - if(FileIO::feof(m_ReadFileHandle)) - RETURNCORRUPT("Invalid truncated ASCII data section"); - - uint64_t length = 0; - - c = '0'; - - while(!FileIO::feof(m_ReadFileHandle) && c != '\n') - { - c = '0'; - FileIO::fread(&c, 1, 1, m_ReadFileHandle); - - if(c == '\n') - break; - - length *= 10; - length += int(c - '0'); - } - - if(FileIO::feof(m_ReadFileHandle)) - RETURNCORRUPT("Invalid truncated ASCII data section"); - - union - { - uint32_t u32; - SectionType t; - } type; - - type.u32 = 0; - - c = '0'; - - while(!FileIO::feof(m_ReadFileHandle) && c != '\n') - { - c = '0'; - FileIO::fread(&c, 1, 1, m_ReadFileHandle); - - if(c == '\n') - break; - - type.u32 *= 10; - type.u32 += int(c - '0'); - } - - if(FileIO::feof(m_ReadFileHandle)) - RETURNCORRUPT("Invalid truncated ASCII data section"); - - string name; - - c = 0; - - while(!FileIO::feof(m_ReadFileHandle) && c != '\n') - { - c = 0; - FileIO::fread(&c, 1, 1, m_ReadFileHandle); - - if(c == 0 || c == '\n') - break; - - name.push_back(c); - } - - if(FileIO::feof(m_ReadFileHandle)) - RETURNCORRUPT("Invalid truncated ASCII data section"); - - Section *sect = new Section(); - sect->flags = eSectionFlag_ASCIIStored; - sect->type = type.t; - sect->name = name; - sect->size = length; - sect->data.resize((size_t)length); - sect->fileoffset = FileIO::ftell64(m_ReadFileHandle); - - FileIO::fread(§->data[0], 1, (size_t)length, m_ReadFileHandle); - - if(sect->type != eSectionType_Unknown && sect->type < eSectionType_Num) - m_KnownSections[sect->type] = sect; - m_Sections.push_back(sect); - } - else if(sectionHeader.isASCII == 0x0) - { - FileIO::fread(reading, 1, offsetof(BinarySectionHeader, name) - 1, m_ReadFileHandle); - - Section *sect = new Section(); - sect->flags = sectionHeader.sectionFlags; - sect->type = sectionHeader.sectionType; - sect->name.resize(sectionHeader.sectionNameLength - 1); - sect->size = sectionHeader.sectionLength; - - FileIO::fread(§->name[0], 1, sectionHeader.sectionNameLength - 1, m_ReadFileHandle); - char nullterm = 0; - FileIO::fread(&nullterm, 1, 1, m_ReadFileHandle); - - sect->fileoffset = FileIO::ftell64(m_ReadFileHandle); - - if(sect->flags & eSectionFlag_LZ4Compressed) - { - sect->compressedReader = new CompressedFileIO(m_ReadFileHandle); - FileIO::fread(§->size, 1, sizeof(uint64_t), m_ReadFileHandle); - - sect->fileoffset += sizeof(uint64_t); - } - - if(sect->type != eSectionType_Unknown && sect->type < eSectionType_Num) - m_KnownSections[sect->type] = sect; - m_Sections.push_back(sect); - - // if section isn't frame capture data and is small enough, read it all into memory now, - // otherwise skip - if(sect->type != eSectionType_FrameCapture && sectionHeader.sectionLength < 4 * 1024 * 1024) - { - sect->data.resize(sectionHeader.sectionLength); - FileIO::fread(§->data[0], 1, sectionHeader.sectionLength, m_ReadFileHandle); - } - else - { - if(sectionHeader.sectionLength == 0xffffffff) - { - RDCWARN( - "Section length 0xFFFFFFFF - assuming truncated value! Seeking to end of file, " - "discarding any remaining sections."); - FileIO::fseek64(m_ReadFileHandle, 0, SEEK_END); - } - else - { - FileIO::fseek64(m_ReadFileHandle, sectionHeader.sectionLength, SEEK_CUR); - } - } - } - else - { - RETURNCORRUPT("Unrecognised section type '%hhx'", sectionHeader.isASCII); - } - } + m_StructuredFile->buffers.push_back(alloc); } else + { + m_Read->SkipBytes(chunkBytes); + } + } +} + +template <> +void Serialiser::EndChunk() +{ + if(ExportStructure()) + { + RDCASSERTMSG("Object Stack is imbalanced!", m_StructureStack.size() <= 1, + m_StructureStack.size()); + + if(!m_StructureStack.empty()) + { + m_StructureStack.back()->type.byteSize = m_ChunkMetadata.length; + m_StructureStack.pop_back(); + } + } + + // only skip remaining bytes if we have a valid length - if we have a length of 0 we wrote this + // chunk in 'streaming mode' (see SetStreamingMode and the Writing EndChunk() impl) so there's + // nothing to skip. + if(m_ChunkMetadata.length > 0) + { + // this will be a no-op if the last chunk length was accurate. If it was a + // conservative estimate of the length then we'll skip some padding bytes + uint64_t readBytes = m_Read->GetOffset() - m_LastChunkOffset; + + if(m_ChunkMetadata.length < readBytes) { RDCERR( - "Capture file from wrong version. This program is logfile version %llu, file is logfile " - "version %llu", - SERIALISE_VERSION, header.version); - - m_ErrorCode = eSerError_UnsupportedVersion; - m_HasError = true; - FileIO::fclose(m_ReadFileHandle); - m_ReadFileHandle = 0; - return; - } - - if(m_KnownSections[eSectionType_FrameCapture] == NULL) - { - RDCERR("Capture file doesn't have a frame capture"); - - m_ErrorCode = eSerError_Corrupt; - m_HasError = true; - FileIO::fclose(m_ReadFileHandle); - m_ReadFileHandle = 0; - return; - } - - m_BufferSize = m_KnownSections[eSectionType_FrameCapture]->size; - m_CurrentBufferSize = (size_t)RDCMIN(m_BufferSize, (uint64_t)64 * 1024); - m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); - m_ReadOffset = 0; - - FileIO::fseek64(m_ReadFileHandle, m_KnownSections[eSectionType_FrameCapture]->fileoffset, - SEEK_SET); - - // read initial buffer of data - ReadFromFile(0, m_CurrentBufferSize); - } - else - { - m_SerVer = SERIALISE_VERSION; - - if(m_Filename != "") - { - m_BufferSize = 0; - m_BufferHead = m_Buffer = NULL; + "!!! " + "READ %llu BYTES, OVERRUNNING CHUNK LENGTH %u. " + "CAPTURE IS CORRUPTED, OR REPLAY MISMATCHED CAPTURED CHUNK. " + "!!!", + readBytes, m_ChunkMetadata.length); } else { - m_BufferSize = sizeHint; - m_BufferHead = m_Buffer = AllocAlignedBuffer((size_t)m_BufferSize); + m_Read->SkipBytes(size_t(m_ChunkMetadata.length - readBytes)); } } + + // align to the natural chunk alignment + m_Read->AlignTo(); +} + +///////////////////////////////////////////////////////////// +// Write Serialiser functions + +template <> +Serialiser::Serialiser(StreamWriter *writer, Ownership own) +{ + m_Write = writer; + m_Read = NULL; + + m_Ownership = own; +} + +template <> +Serialiser::~Serialiser() +{ + if(m_Ownership == Ownership::Stream && m_Write) + { + m_Write->Finish(); + delete m_Write; + } } -void Serialiser::Reset() +template <> +void Serialiser::SetChunkMetadataRecording(uint32_t flags) { - if(m_ResolverThread != 0) - { - m_ResolverThreadKillSignal = true; + // cannot change this mid-chunk + RDCASSERT(m_Write->GetOffset() == 0); - Threading::JoinThread(m_ResolverThread); - Threading::CloseThread(m_ResolverThread); - m_ResolverThread = 0; - } - - m_pUserData = NULL; - - m_DebugText = ""; - m_DebugTextWriting = false; - - RDCEraseEl(m_KnownSections); - - m_HasError = false; - m_ErrorCode = eSerError_None; - - m_Mode = NONE; - - m_Indent = 0; - - SAFE_DELETE(m_pCallstack); - SAFE_DELETE(m_pResolver); - if(m_Buffer) - { - FreeAlignedBuffer(m_Buffer); - m_Buffer = NULL; - } - - m_ChunkLookup = NULL; - - m_AlignedData = false; - - m_ReadFileHandle = NULL; - - m_ReadOffset = 0; - - m_BufferHead = m_Buffer = NULL; - m_CurrentBufferSize = 0; - m_BufferSize = 0; + m_ChunkFlags = flags; } -Serialiser::~Serialiser() +template <> +uint32_t Serialiser::BeginChunk(uint32_t chunkID, uint32_t byteLength) { - if(m_ResolverThread != 0) { - m_ResolverThreadKillSignal = true; - Threading::JoinThread(m_ResolverThread); - Threading::CloseThread(m_ResolverThread); - m_ResolverThread = 0; - } + // chunk index needs to be valid + RDCASSERT(chunkID > 0); - if(m_ReadFileHandle) - { - FileIO::fclose(m_ReadFileHandle); - m_ReadFileHandle = 0; - } - - for(size_t i = 0; i < m_Sections.size(); i++) - { - SAFE_DELETE(m_Sections[i]->compressedReader); - SAFE_DELETE(m_Sections[i]); - } - - for(size_t i = 0; i < m_Chunks.size(); i++) - { - if(m_Chunks[i]->IsTemporary()) - SAFE_DELETE(m_Chunks[i]); - } - - m_Chunks.clear(); - - SAFE_DELETE(m_pResolver); - SAFE_DELETE(m_pCallstack); - if(m_Buffer) - { - FreeAlignedBuffer(m_Buffer); - m_Buffer = NULL; - } - m_Buffer = NULL; - m_BufferHead = NULL; -} - -void Serialiser::WriteBytes(const byte *buf, size_t nBytes) -{ - if(m_HasError) - { - RDCERR("Writing bytes with error state serialiser"); - return; - } - - if(m_Buffer + m_BufferSize < m_BufferHead + nBytes + 8) - { - // reallocate - while(m_Buffer + m_BufferSize < m_BufferHead + nBytes + 8) { - m_BufferSize += 128 * 1024; - } + uint32_t c = chunkID & ChunkIndexMask; + RDCASSERT(chunkID <= ChunkIndexMask); - byte *newBuf = AllocAlignedBuffer((size_t)m_BufferSize); + c |= m_ChunkFlags; - size_t curUsed = m_BufferHead - m_Buffer; - - memcpy(newBuf, m_Buffer, curUsed); - - FreeAlignedBuffer(m_Buffer); - - m_Buffer = newBuf; - m_BufferHead = newBuf + curUsed; - } - - memcpy(m_BufferHead, buf, nBytes); - - m_BufferHead += nBytes; -} - -void *Serialiser::ReadBytes(size_t nBytes) -{ - if(m_HasError) - { - RDCERR("Reading bytes with error state serialiser"); - return NULL; - } - - // if we would read off the end of our current window - if(m_BufferHead + nBytes > m_Buffer + m_CurrentBufferSize) - { - // store old buffer and the read data, so we can move it into the new buffer - byte *oldBuffer = m_Buffer; - - // always keep at least a certain window behind what we read. - size_t backwardsWindow = RDCMIN((size_t)64, size_t(m_BufferHead - m_Buffer)); - - byte *currentData = m_BufferHead - backwardsWindow; - size_t currentDataSize = m_CurrentBufferSize - (m_BufferHead - m_Buffer) + backwardsWindow; - - size_t BufferOffset = m_BufferHead - m_Buffer; - - // if we are reading more than our current buffer size, expand the buffer size - if(nBytes + backwardsWindow > m_CurrentBufferSize) - { - // very conservative resizing - don't do "double and add" - to avoid - // a 1GB buffer being read and needing to allocate 2GB. The cost is we - // will reallocate a bit more often - m_CurrentBufferSize = nBytes + backwardsWindow; - m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); - } - - // move the unread data into the buffer - memmove(m_Buffer, currentData, currentDataSize); - - if(BufferOffset > backwardsWindow) - { - m_ReadOffset += BufferOffset - backwardsWindow; - m_BufferHead = m_Buffer + backwardsWindow; - } - else - { - m_BufferHead = m_Buffer + BufferOffset; - } - - // if there's anything left of the file to read in, do so now - ReadFromFile(currentDataSize, RDCMIN(m_CurrentBufferSize - currentDataSize, - size_t(m_BufferSize - m_ReadOffset - currentDataSize))); - - if(oldBuffer != m_Buffer) - FreeAlignedBuffer(oldBuffer); - } - - void *ret = m_BufferHead; - - m_BufferHead += nBytes; - - RDCASSERT(m_BufferHead <= m_Buffer + m_CurrentBufferSize); - - return ret; -} - -void Serialiser::ReadFromFile(uint64_t bufferOffs, size_t length) -{ - RDCASSERT(m_ReadFileHandle); - - if(m_ReadFileHandle == NULL) - return; - - Section *s = m_KnownSections[eSectionType_FrameCapture]; - - RDCASSERT(s); - - if(s->flags & eSectionFlag_LZ4Compressed) - { - RDCASSERT(s->compressedReader); - s->compressedReader->Read(m_Buffer + bufferOffs, length); - } - else - { - FileIO::fread(m_Buffer + bufferOffs, 1, length, m_ReadFileHandle); - } -} - -void Serialiser::SetPersistentBlock(uint64_t offs) -{ - // as long as this is called immediately after pushing the chunk context at the - // offset, we will always have the start in memory, as we keep 64 bytes of - // a backwards window even if we had to shift the currently in-memory bytes - // while reading the chunk header - RDCASSERT(m_ReadOffset <= offs); - - // also can't persistent block ahead of where we are - RDCASSERT(offs < (m_BufferHead - m_Buffer) + m_ReadOffset); - - // ensure sane offset - RDCASSERT(offs < m_BufferSize); - - size_t persistentSize = (size_t)(m_BufferSize - offs); - - // allocate our persistent buffer - byte *newBuf = AllocAlignedBuffer(persistentSize); - - // save where buffer head was as file-offset - uint64_t prevOffs = uint64_t(m_BufferHead - m_Buffer) + m_ReadOffset; - - // find the range of the persistent block that we have in memory - byte *persistentBase = m_Buffer + (offs - m_ReadOffset); - size_t persistentInMemory = - RDCMIN(persistentSize, size_t(m_CurrentBufferSize - (offs - m_ReadOffset))); - - memcpy(newBuf, persistentBase, persistentInMemory); - - FreeAlignedBuffer(m_Buffer); - - m_CurrentBufferSize = persistentSize; - m_Buffer = newBuf; - m_ReadOffset = offs; - - // set the head back to where it was - m_BufferHead = m_Buffer + (prevOffs - offs); - - // if we didn't read everything, read the rest - if(persistentInMemory < persistentSize) - { - ReadFromFile(persistentInMemory, persistentSize - persistentInMemory); - } - - RDCASSERT(m_ReadFileHandle); - - // close the file handle - FileIO::fclose(m_ReadFileHandle); - m_ReadFileHandle = 0; -} - -void Serialiser::SetOffset(uint64_t offs) -{ - if(m_HasError) - { - RDCERR("Setting offset with error state serialiser"); - return; - } - - // if we're jumping back before our in-memory window just reset the window - // and load it all in from scratch. - if(m_Mode == READING && offs < m_ReadOffset) - { - // if we're reading from file, only support rewinding all the way to the start - RDCASSERT(m_ReadFileHandle == NULL || offs == 0); - - if(m_ReadFileHandle) - { - Section *s = m_KnownSections[eSectionType_FrameCapture]; - RDCASSERT(s); - FileIO::fseek64(m_ReadFileHandle, s->fileoffset, SEEK_SET); - - if(s->flags & eSectionFlag_LZ4Compressed) - { - RDCASSERT(s->compressedReader); - s->compressedReader->Reset(); - } - } - - FreeAlignedBuffer(m_Buffer); - - m_CurrentBufferSize = (size_t)RDCMIN(m_BufferSize, (uint64_t)64 * 1024); - m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); - m_ReadOffset = offs; - - ReadFromFile(0, m_CurrentBufferSize); - } - - RDCASSERT(m_BufferHead && m_Buffer && offs <= GetSize()); - m_BufferHead = m_Buffer + offs - m_ReadOffset; - m_Indent = 0; -} - -void Serialiser::InitCallstackResolver() -{ - if(m_pResolver == NULL && m_ResolverThread == 0 && - m_KnownSections[eSectionType_ResolveDatabase] != NULL) - { - m_ResolverThreadKillSignal = false; - m_ResolverThread = Threading::CreateThread([this]() { CreateResolver(); }); - } -} - -void Serialiser::SetCallstack(uint64_t *levels, size_t numLevels) -{ - if(m_pCallstack == NULL) - m_pCallstack = Callstack::Create(); - - m_pCallstack->Set(levels, numLevels); -} - -void Serialiser::CreateResolver() -{ - string dir = dirname(m_Filename); - - Section *s = m_KnownSections[Serialiser::eSectionType_ResolveDatabase]; - RDCASSERT(s); - - m_pResolver = - Callstack::MakeResolver((char *)&s->data[0], s->data.size(), dir, &m_ResolverThreadKillSignal); -} - -void Serialiser::FlushToDisk() -{ - SCOPED_TIMER("File writing"); - - if(m_Filename != "" && !m_HasError && m_Mode == WRITING) - { - RDCDEBUG("writing capture files"); - - if(m_DebugEnabled && !m_DebugText.empty()) - { - FILE *dbgFile = FileIO::fopen((m_Filename + ".txt").c_str(), "wb"); - - if(!dbgFile) - { - RDCERR("Can't open debug capture file '%s'", (m_Filename + ".txt").c_str()); - } - else - { - const char *str = m_DebugText.c_str(); - size_t len = m_DebugText.length(); - const size_t chunkSize = 10 * 1024 * 1024; - while(len > 0) - { - size_t writeSize = RDCMIN(len, chunkSize); - size_t written = FileIO::fwrite(str, 1, writeSize, dbgFile); - - RDCASSERT(written == writeSize); - - str += writeSize; - len -= writeSize; - } - - FileIO::fclose(dbgFile); - } - } - - FILE *binFile = FileIO::fopen(m_Filename.c_str(), "w+b"); - - if(!binFile) - { - RDCERR("Can't open capture file '%s' for write, errno %d", m_Filename.c_str(), errno); - m_ErrorCode = eSerError_FileIO; - m_HasError = true; - return; - } - - RDCDEBUG("Opened capture file for write"); - - FileHeader header; // automagically initialised with correct data - - // write header - FileIO::fwrite(&header, 1, sizeof(FileHeader), binFile); - - static const byte padding[BufferAlignment] = {0}; - - uint64_t compressedSizeOffset = 0; - uint64_t uncompressedSizeOffset = 0; - - // write frame capture section header - { - const char sectionName[] = "renderdoc/internal/framecapture"; - - BinarySectionHeader section = {0}; - section.isASCII = 0; // redundant but explicit - section.sectionNameLength = sizeof(sectionName); // includes null terminator - section.sectionType = eSectionType_FrameCapture; - section.sectionFlags = eSectionFlag_LZ4Compressed; - section.sectionLength = - 0; // will be fixed up later, to avoid having to compress everything into memory - - compressedSizeOffset = FileIO::ftell64(binFile) + offsetof(BinarySectionHeader, sectionLength); - - FileIO::fwrite(§ion, 1, offsetof(BinarySectionHeader, name), binFile); - FileIO::fwrite(sectionName, 1, sizeof(sectionName), binFile); - - uint64_t len = 0; // will be fixed up later - uncompressedSizeOffset = FileIO::ftell64(binFile); - FileIO::fwrite(&len, 1, sizeof(uint64_t), binFile); - } - - CompressedFileIO fwriter(binFile); - - // track offset so we can add padding. The padding is relative - // to the start of the decompressed buffer, so we start it from 0 - uint64_t offs = 0; - uint64_t alignedoffs = 0; - - // write frame capture contents - for(size_t i = 0; i < m_Chunks.size(); i++) - { - Chunk *chunk = m_Chunks[i]; - - alignedoffs = AlignUp(offs, BufferAlignment); - - if(offs != alignedoffs && chunk->IsAligned()) - { - uint16_t chunkIdx = 0; // write a '0' chunk that indicates special behaviour - fwriter.Write(&chunkIdx, sizeof(chunkIdx)); - offs += sizeof(chunkIdx); - - uint8_t controlByte = 0; // control byte 0 indicates padding - fwriter.Write(&controlByte, sizeof(controlByte)); - offs += sizeof(controlByte); - - offs++; // we will have to write out a byte indicating how much padding exists, so add 1 - alignedoffs = AlignUp(offs, BufferAlignment); - - RDCCOMPILE_ASSERT(BufferAlignment < 0x100, - "Buffer alignment must be less than 256"); // with a byte at most - // indicating how many bytes - // to pad, - // this is our maximal representable alignment - - uint8_t padLength = (alignedoffs - offs) & 0xff; - fwriter.Write(&padLength, sizeof(padLength)); - - // we might have padded with the control bytes, so only write some bytes if we need to - if(padLength > 0) - { - fwriter.Write(padding, size_t(alignedoffs - offs)); - offs += alignedoffs - offs; - } - } - - fwriter.Write(chunk->GetData(), chunk->GetLength()); - - offs += chunk->GetLength(); - - if(chunk->IsTemporary()) - SAFE_DELETE(chunk); - } - - fwriter.Flush(); - - m_Chunks.clear(); - - // fixup section size - { - uint32_t compsize = 0; - uint64_t uncompsize = 0; - - uint64_t curoffs = FileIO::ftell64(binFile); - - FileIO::fseek64(binFile, compressedSizeOffset, SEEK_SET); - - uint64_t realCompSize = fwriter.GetCompressedSize(); - - if(realCompSize > 0xffffffff) - { - RDCERR("Compressed file size %llu exceeds representable capture size! May cause corruption", - realCompSize); - realCompSize = 0xffffffffULL; - } - - compsize = uint32_t(realCompSize); - FileIO::fwrite(&compsize, 1, sizeof(compsize), binFile); - - FileIO::fseek64(binFile, uncompressedSizeOffset, SEEK_SET); - - uncompsize = fwriter.GetUncompressedSize(); - FileIO::fwrite(&uncompsize, 1, sizeof(uncompsize), binFile); - - FileIO::fseek64(binFile, curoffs, SEEK_SET); - - RDCLOG("Compressed frame capture data from %llu to %llu", fwriter.GetUncompressedSize(), - fwriter.GetCompressedSize()); - } - - char *symbolDB = NULL; - size_t symbolDBSize = 0; - - if(RenderDoc::Inst().GetCaptureOptions().CaptureCallstacks || - RenderDoc::Inst().GetCaptureOptions().CaptureCallstacksOnlyDraws) - { - // get symbol database - Callstack::GetLoadedModules(symbolDB, symbolDBSize); - - symbolDB = new char[symbolDBSize]; - symbolDBSize = 0; - - Callstack::GetLoadedModules(symbolDB, symbolDBSize); - } - - // write symbol database section - if(symbolDB) - { - const char sectionName[] = "renderdoc/internal/resolvedb"; - - BinarySectionHeader section = {0}; - section.isASCII = 0; // redundant but explicit - section.sectionNameLength = sizeof(sectionName); // includes null terminator - section.sectionType = eSectionType_ResolveDatabase; - section.sectionLength = (uint32_t)symbolDBSize; - - FileIO::fwrite(§ion, 1, offsetof(BinarySectionHeader, name), binFile); - FileIO::fwrite(sectionName, 1, sizeof(sectionName), binFile); - - // write actual data - FileIO::fwrite(symbolDB, 1, symbolDBSize, binFile); - - SAFE_DELETE_ARRAY(symbolDB); - } - - // write the machine identifier as an ASCII section - { - const char sectionName[] = "renderdoc/internal/machineid"; - - uint64_t machineID = OSUtility::GetMachineIdent(); - - BinarySectionHeader section = {0}; - section.isASCII = 0; // redundant but explicit - section.sectionNameLength = sizeof(sectionName); // includes null terminator - section.sectionType = eSectionType_MachineID; - section.sectionFlags = eSectionFlag_None; - section.sectionLength = sizeof(machineID); - - FileIO::fwrite(§ion, 1, offsetof(BinarySectionHeader, name), binFile); - FileIO::fwrite(sectionName, 1, sizeof(sectionName), binFile); - FileIO::fwrite(&machineID, 1, sizeof(machineID), binFile); - } - - FileIO::fclose(binFile); - } -} - -void Serialiser::DebugPrint(const char *fmt, ...) -{ - if(m_HasError) - { - RDCERR("Debug printing with error state serialiser"); - return; - } - - char tmpBuf[1024]; - - va_list args; - va_start(args, fmt); - StringFormat::vsnprintf(tmpBuf, 1023, fmt, args); - tmpBuf[1023] = '\0'; - va_end(args); - - m_DebugText += GetIndent(); - m_DebugText += tmpBuf; -} - -uint32_t Serialiser::PushContext(const char *name, const char *typeName, uint32_t chunkIdx, - bool smallChunk) -{ - // if writing, and chunkidx isn't 0 (debug non-scope), then either we're nested - // or we should be writing into the start of the serialiser. A serialiser should - // only ever have one chunk in it - RDCASSERT(m_Mode < WRITING || m_Indent > 0 || GetOffset() == 0 || chunkIdx == 0); - - // we should not be pushing contexts directly into a file serialiser - RDCASSERT(m_Mode < WRITING || m_Filename.empty()); - - if(m_Mode >= WRITING) - { - if(chunkIdx > 0) - { - uint16_t c = chunkIdx & 0x3fff; - RDCASSERT(chunkIdx <= 0x3fff); + m_ChunkMetadata.chunkID = chunkID; ///////////////// - Callstack::Stackwalk *call = NULL; + m_Write->Write(c); - if(m_Indent == 0) + if(c & ChunkCallstack) { - if(RenderDoc::Inst().GetCaptureOptions().CaptureCallstacks && - !RenderDoc::Inst().GetCaptureOptions().CaptureCallstacksOnlyDraws) + if(m_ChunkMetadata.callstack.empty()) { - call = Callstack::Collect(); + bool collect = RenderDoc::Inst().GetCaptureOptions().CaptureCallstacks; - RDCASSERT(call->NumLevels() < 0xff); - } - } + if(RenderDoc::Inst().GetCaptureOptions().CaptureCallstacksOnlyDraws) + collect = collect && m_DrawChunk; - if(call) - c |= 0x8000; - if(smallChunk) - c |= 0x4000; - - WriteFrom(c); - - if(call) - { - uint8_t numLevels = call->NumLevels() & 0xff; - WriteFrom(numLevels); - - if(call->NumLevels()) - { - WriteBytes((byte *)call->GetAddrs(), sizeof(uint64_t) * numLevels); - } - - SAFE_DELETE(call); - } - - // will be fixed up in PopContext - if(smallChunk) - { - uint16_t chunkSize = 0xbeeb; - m_ChunkFixups.push_back(0x8000000000000000ULL | GetOffset()); - WriteFrom(chunkSize); - } - else - { - uint32_t chunkSize = 0xbeebfeed; - m_ChunkFixups.push_back(GetOffset() & ~0x8000000000000000ULL); - WriteFrom(chunkSize); - } - } - - if(m_DebugTextWriting) - { - if(typeName) - DebugPrint("%s = %s (%d)\n", name, typeName, chunkIdx); - else - DebugPrint("%s (%d)\n", name, chunkIdx); - DebugPrint("{\n"); - } - } - else - { - if(m_Indent == 0) - { - // reset debug text - m_DebugText = ""; - } - - if(chunkIdx > 0) - { - uint16_t c = 0; - ReadInto(c); - - // chunk index 0 is not allowed in normal situations. - // allows us to indicate some control bytes - while(c == 0) - { - uint8_t *controlByte = (uint8_t *)ReadBytes(1); - - if(*controlByte == 0x0) - { - // padding - uint8_t *padLength = (uint8_t *)ReadBytes(1); - - // might have padded with these 5 control bytes, - // so a pad length of 0 IS VALID. - if(*padLength > 0) + if(collect) { - ReadBytes((size_t)*padLength); + Callstack::Stackwalk *stack = Callstack::Collect(); + if(stack && stack->NumLevels() > 0) + { + m_ChunkMetadata.callstack.assign(stack->GetAddrs(), stack->NumLevels()); + } + + SAFE_DELETE(stack); } } - else - { - RDCERR("Unexpected control byte: %x", (uint32_t)*controlByte); - } - ReadInto(c); + uint32_t numFrames = (uint32_t)m_ChunkMetadata.callstack.size(); + m_Write->Write(numFrames); + + m_Write->Write(m_ChunkMetadata.callstack.data(), m_ChunkMetadata.callstack.byteSize()); } - chunkIdx = c & 0x3fff; - bool callstack = (c & 0x8000) > 0; - bool smallchunk = (c & 0x4000) > 0; - - ///////////////// - - if(m_Indent == 0) + if(c & ChunkThreadID) { - if(callstack) - { - uint8_t callLen = 0; - ReadInto(callLen); + if(m_ChunkMetadata.threadID == 0) + m_ChunkMetadata.threadID = Threading::GetCurrentID(); - uint64_t *calls = (uint64_t *)ReadBytes(callLen * sizeof(uint64_t)); - SetCallstack(calls, callLen); - } - else - { - SetCallstack(NULL, 0); - } + m_Write->Write(m_ChunkMetadata.threadID); } - ///////////////// + if(c & ChunkDuration) + m_Write->Write(m_ChunkMetadata.durationMicro); - if(smallchunk) + if(c & ChunkTimestamp) { - uint16_t miniSize = 0xbeeb; - ReadInto(miniSize); + if(m_ChunkMetadata.timestampMicro == 0) + m_ChunkMetadata.timestampMicro = RenderDoc::Inst().GetMicrosecondTimestamp(); - m_LastChunkLen = miniSize; + m_Write->Write(m_ChunkMetadata.timestampMicro); + } + + if(byteLength > 0 || m_DataStreaming) + { + // write length, assuming it is an upper bound + m_ChunkFixup = 0; + m_Write->Write(byteLength); + m_LastChunkOffset = m_Write->GetOffset(); + m_ChunkMetadata.length = byteLength; } else { + // length will be fixed up in EndChunk uint32_t chunkSize = 0xbeebfeed; - ReadInto(chunkSize); - - m_LastChunkLen = chunkSize; + m_ChunkFixup = m_Write->GetOffset(); + m_Write->Write(chunkSize); } } - - if(!name && m_ChunkLookup) - name = m_ChunkLookup(chunkIdx); - - if(m_DebugTextWriting) - { - if(typeName) - DebugPrint("%s = %s\n", name ? name : "Unknown", typeName); - else - DebugPrint("%s\n", name ? name : "Unknown"); - DebugPrint("{\n"); - } } - m_Indent++; - - return chunkIdx; -} - -void Serialiser::PopContext(uint32_t chunkIdx) -{ - m_Indent = RDCMAX(m_Indent - 1, 0); - - if(m_Mode >= WRITING) - { - if(chunkIdx > 0 && m_Mode == WRITING) - { - // fix up the latest PushContext (guaranteed to match this one as Pushes and Pops match) - RDCASSERT(!m_ChunkFixups.empty()); - - uint64_t chunkOffset = m_ChunkFixups.back(); - m_ChunkFixups.pop_back(); - - bool smallchunk = (chunkOffset & 0x8000000000000000ULL) > 0; - chunkOffset &= ~0x8000000000000000ULL; - - uint64_t curOffset = GetOffset(); - - RDCASSERT(curOffset > chunkOffset); - - uint64_t chunkLength = - (curOffset - chunkOffset) - (smallchunk ? sizeof(uint16_t) : sizeof(uint32_t)); - - RDCASSERT(chunkLength < 0xffffffff); - - uint32_t chunklen = (uint32_t)chunkLength; - - byte *head = m_BufferHead; - SetOffset(chunkOffset); - if(smallchunk) - { - uint16_t miniSize = (chunklen & 0xffff); - RDCASSERT(chunklen <= 0xffff); - WriteFrom(miniSize); - } - else - { - WriteFrom(chunklen); - } - m_BufferHead = head; - } - - if(m_DebugTextWriting) - DebugPrint("}\n"); - } - else - { - if(m_DebugTextWriting) - DebugPrint("}\n"); - } -} - -///////////////////////////////////////////////////////////// -// Serialise functions - -///////////////////////////////////////////////////////////// -// generic - -void Serialiser::SerialiseString(const char *name, string &el) -{ - uint32_t len = (uint32_t)el.length(); - - Serialise(NULL, len); - - if(m_Mode == READING) - el.resize(len); - - if(m_Mode >= WRITING) - { - WriteBytes((byte *)el.c_str(), len); - - if(m_DebugTextWriting) - { - string s = el; - if(s.length() > 64) - s = s.substr(0, 60) + "..."; - DebugPrint("%s: \"%s\"\n", name, s.c_str()); - } - } - else - { - memcpy(&el[0], ReadBytes(len), len); - - if(m_DebugTextWriting) - { - string s = el; - if(s.length() > 64) - s = s.substr(0, 60) + "..."; - DebugPrint("%s: \"%s\"\n", name, s.c_str()); - } - } -} - -void Serialiser::Insert(Chunk *chunk) -{ - m_Chunks.push_back(chunk); - - m_DebugText += chunk->GetDebugString(); -} - -void Serialiser::AlignNextBuffer(const size_t alignment) -{ - // on new logs, we don't have to align. This code will be deleted once backwards-compat is dropped - if(m_Mode >= WRITING || m_SerVer >= 0x00000032) - return; - - // this is a super hack but it's the easiest way to align a buffer to a larger pow2 alignment - // than the default 16-bytes, while still able to be backwards compatible with old logs that - // weren't so aligned. We know that SerialiseBuffer will align to the nearest 16-byte boundary - // after serialising 4 bytes of length, so we pad up to exactly 4 bytes before the desired - // alignment, then after the 4 byte length there's nothing for the other padding to do. - // - // Note the chunk still needs to be aligned when the memory is allocated - this just ensures - // the offset from the start is also aligned - - uint32_t len = 0; - - if(m_Mode >= WRITING) - { - // add sizeof(uint32_t) since we'll be serialising out how much padding is here, - // then another sizeof(uint32_t) so we're aligning the offset after the buffer's - // serialised length - uint64_t curoffs = GetOffset() + sizeof(uint32_t) * 2; - uint64_t alignedoffs = AlignUp(curoffs, (uint64_t)alignment); - - len = uint32_t(alignedoffs - curoffs); - } - - // avoid dynamically allocating - RDCASSERT(alignment <= 128); - byte padding[128] = {0}; - - if(m_Mode >= WRITING) - { - WriteFrom(len); - WriteBytes(&padding[0], (size_t)len); - } - else - { - ReadInto(len); - ReadBytes(len); - } -} - -void Serialiser::SerialiseBuffer(const char *name, byte *&buf, size_t &len) -{ - uint32_t bufLen = (uint32_t)len; - - if(m_Mode >= WRITING) - { - WriteFrom(bufLen); - - // ensure byte alignment - uint64_t offs = GetOffset(); - uint64_t alignedoffs = AlignUp(offs, BufferAlignment); - - if(offs != alignedoffs) - { - static const byte padding[BufferAlignment] = {0}; - WriteBytes(&padding[0], (size_t)(alignedoffs - offs)); - } - - RDCASSERT((GetOffset() % BufferAlignment) == 0); - - WriteBytes(buf, bufLen); - - m_AlignedData = true; - } - else - { - ReadInto(bufLen); - - // ensure byte alignment - uint64_t offs = GetOffset(); - - // serialise version 0x00000031 had only 16-byte alignment - uint64_t alignedoffs = AlignUp(offs, m_SerVer == 0x00000031 ? 16 : BufferAlignment); - - if(offs != alignedoffs) - { - ReadBytes((size_t)(alignedoffs - offs)); - } - - if(buf == NULL) - buf = new byte[bufLen]; - memcpy(buf, ReadBytes(bufLen), bufLen); - } - - len = (size_t)bufLen; - - if(m_DebugTextWriting && name && name[0]) - { - const char *ellipsis = "..."; - - uint32_t lbuf[4]; - - memcpy(lbuf, buf, RDCMIN(len, 4 * sizeof(uint32_t))); - - if(bufLen <= 16) - { - ellipsis = " "; - } - - DebugPrint("%s: RawBuffer % 5d:< 0x%08x 0x%08x 0x%08x 0x%08x %s>\n", name, bufLen, lbuf[0], - lbuf[1], lbuf[2], lbuf[3], ellipsis); - } + return chunkID; } template <> -void Serialiser::Serialise(const char *name, string &el) +void Serialiser::EndChunk() { - SerialiseString(name, el); -} + m_DrawChunk = false; -// floats need aligned reads -template <> -void Serialiser::ReadInto(float &f) -{ - if(m_HasError) + if(m_DataStreaming) { - RDCERR("Reading into with error state serialiser"); - return; + // nothing to fixup, length is unused + } + else if(m_ChunkFixup != 0) + { + // fix up the chunk header + uint64_t chunkOffset = m_ChunkFixup; + m_ChunkFixup = 0; + + uint64_t curOffset = m_Write->GetOffset(); + + RDCASSERT(curOffset > chunkOffset); + + uint64_t chunkLength = (curOffset - chunkOffset) - sizeof(uint32_t); + + RDCASSERT(chunkLength < 0xffffffff); + + uint32_t chunklen = (uint32_t)chunkLength; + + m_Write->WriteAt(chunkOffset, chunklen); + } + else + { + uint64_t writtenLength = (m_Write->GetOffset() - m_LastChunkOffset); + + if(writtenLength < m_ChunkMetadata.length) + { + uint64_t numPadBytes = m_ChunkMetadata.length - writtenLength; + + // need to write some padding bytes so that the length is accurate + for(uint64_t i = 0; i < numPadBytes; i++) + { + byte padByte = 0xbb; + m_Write->Write(padByte); + } + + RDCDEBUG("Chunk estimated at %u bytes, actual length %llu. Added %llu bytes padding.", + m_ChunkMetadata.length, writtenLength, numPadBytes); + } + else if(writtenLength > m_ChunkMetadata.length) + { + RDCERR( + "!!! " + "ESTIMATED UPPER BOUND CHUNK LENGTH %u EXCEEDED: %llu. " + "CAPTURE WILL BE CORRUPTED. " + "!!!", + m_ChunkMetadata.length, writtenLength); + } + else + { + RDCDEBUG("Chunk was exactly the estimate of %u bytes.", m_ChunkMetadata.length); + } } - char *data = (char *)ReadBytes(sizeof(float)); + // align to the natural chunk alignment + m_Write->AlignTo(); - memcpy(&f, data, sizeof(float)); + m_ChunkMetadata = SDChunkMetaData(); } -///////////////////////////////////////////////////////////// -// String conversions for debug log. +template <> +void Serialiser::WriteStructuredFile(const SDFile &file) +{ + Serialiser scratchWriter( + new StreamWriter(StreamWriter::DefaultScratchSize), Ownership::Stream); + + // slightly cheeky to cast away the const, but we don't modify it in a writing serialiser + scratchWriter.m_StructuredFile = m_StructuredFile = (SDFile *)&file; + + for(size_t i = 0; i < file.chunks.size(); i++) + { + const SDChunk &chunk = *file.chunks[i]; + + m_ChunkMetadata = chunk.metadata; + + m_ChunkFlags = 0; + + if(!m_ChunkMetadata.callstack.empty()) + m_ChunkFlags |= ChunkCallstack; + + if(m_ChunkMetadata.threadID != 0) + m_ChunkFlags |= ChunkThreadID; + + if(m_ChunkMetadata.durationMicro != 0) + m_ChunkFlags |= ChunkDuration; + + if(m_ChunkMetadata.timestampMicro != 0) + m_ChunkFlags |= ChunkTimestamp; + + Serialiser *ser = this; + + if(m_ChunkMetadata.length == 0) + { + ser = &scratchWriter; + scratchWriter.m_ChunkMetadata = m_ChunkMetadata; + scratchWriter.m_ChunkFlags = m_ChunkFlags; + } + + ser->BeginChunk(m_ChunkMetadata.chunkID, m_ChunkMetadata.length); + + if(chunk.metadata.flags & SDChunkFlags::OpaqueChunk) + { + RDCASSERT(chunk.data.children.size() == 1); + + size_t bufID = (size_t)chunk.data.children[0]->data.basic.u; + byte *ptr = m_StructuredFile->buffers[bufID]->data(); + size_t len = m_StructuredFile->buffers[bufID]->size(); + + ser->GetWriter()->Write(ptr, len); + } + else + { + for(size_t o = 0; o < chunk.data.children.size(); o++) + { + // note, we don't need names because we aren't exporting structured data + ser->Serialise("", chunk.data.children[o]); + } + } + + ser->EndChunk(); + + if(m_ChunkMetadata.length == 0) + { + m_Write->Write(scratchWriter.GetWriter()->GetData(), scratchWriter.GetWriter()->GetOffset()); + scratchWriter.GetWriter()->Rewind(); + } + } + + m_StructuredFile = &m_StructData; + scratchWriter.m_StructuredFile = &scratchWriter.m_StructData; +} + +template <> +std::string DoStringise(const SDBasic &el) +{ + BEGIN_ENUM_STRINGISE(SDBasic); + { + STRINGISE_ENUM_CLASS(Chunk); + STRINGISE_ENUM_CLASS(Struct); + STRINGISE_ENUM_CLASS(Array); + STRINGISE_ENUM_CLASS(Null); + STRINGISE_ENUM_CLASS(Buffer); + STRINGISE_ENUM_CLASS(String); + STRINGISE_ENUM_CLASS(Enum); + STRINGISE_ENUM_CLASS(UnsignedInteger); + STRINGISE_ENUM_CLASS(SignedInteger); + STRINGISE_ENUM_CLASS(Float); + STRINGISE_ENUM_CLASS(Boolean); + STRINGISE_ENUM_CLASS(Character); + } + END_ENUM_STRINGISE(); +} + +template <> +std::string DoStringise(const SDTypeFlags &el) +{ + BEGIN_BITFIELD_STRINGISE(SDTypeFlags); + { + STRINGISE_BITFIELD_CLASS_VALUE(NoFlags); + + STRINGISE_BITFIELD_CLASS_BIT(HasCustomString); + STRINGISE_BITFIELD_CLASS_BIT(Hidden); + STRINGISE_BITFIELD_CLASS_BIT(Nullable); + STRINGISE_BITFIELD_CLASS_BIT(NullString); + } + END_BITFIELD_STRINGISE(); +} + +template <> +std::string DoStringise(const SDChunkFlags &el) +{ + BEGIN_BITFIELD_STRINGISE(SDChunkFlags); + { + STRINGISE_BITFIELD_CLASS_VALUE(NoFlags); + + STRINGISE_BITFIELD_CLASS_BIT(OpaqueChunk); + } + END_BITFIELD_STRINGISE(); +} + +template +void DoSerialise(SerialiserType &ser, SDType &el) +{ + SERIALISE_MEMBER(name); + SERIALISE_MEMBER(basetype); + SERIALISE_MEMBER(flags); + SERIALISE_MEMBER(byteSize); +} + +template +void DoSerialise(SerialiserType &ser, SDChunkMetaData &el) +{ + SERIALISE_MEMBER(chunkID); + SERIALISE_MEMBER(flags); + SERIALISE_MEMBER(length); + SERIALISE_MEMBER(threadID); + SERIALISE_MEMBER(durationMicro); + SERIALISE_MEMBER(timestampMicro); + SERIALISE_MEMBER(callstack); +} + +template +void DoSerialise(SerialiserType &ser, SDObjectPODData &el) +{ + SERIALISE_MEMBER(u); +} + +template +void DoSerialise(SerialiserType &ser, StructuredObjectList &el) +{ + // since structured objects aren't intended to be exported as nice structured data, only for pure + // transfer purposes, we don't make a proper array here and instead just manually serialise count + // + elements + uint64_t count = el.size(); + ser.Serialise("count", count); + + if(ser.IsReading()) + el.resize((size_t)count); + + for(size_t c = 0; c < (size_t)count; c++) + { + // we also assume that the caller serialising these objects will handle lifetime management. + if(ser.IsReading()) + el[c] = new SDObject("", ""); + + ser.Serialise("$el", *el[c]); + } +} + +template +void DoSerialise(SerialiserType &ser, SDObjectData &el) +{ + SERIALISE_MEMBER(basic); + SERIALISE_MEMBER(str); + SERIALISE_MEMBER(children); +} + +template +void DoSerialise(SerialiserType &ser, SDObject &el) +{ + SERIALISE_MEMBER(name); + SERIALISE_MEMBER(type); + SERIALISE_MEMBER(data); +} + +template +void DoSerialise(SerialiserType &ser, SDChunk &el) +{ + SERIALISE_MEMBER(name); + SERIALISE_MEMBER(type); + SERIALISE_MEMBER(data); + SERIALISE_MEMBER(metadata); +} + +INSTANTIATE_SERIALISE_TYPE(SDChunk); + +// serialise the pointer version - special case for writing a structured file, so can assume writing +template +void DoSerialise(SerialiserType &ser, SDObject *el) +{ + // clang barfs if we try to do ser.IsWriting() here for some reason, claiming it isn't static and + // const enough for static_assert. As a workaround we call IsWriting as a static member of the + // type itself, and that works. + RDCCOMPILE_ASSERT(SerialiserType::IsWriting(), + "SDObject pointer only supported for writing serialisation"); + if(el->type.flags & SDTypeFlags::Nullable) + { + bool present = el->type.basetype != SDBasic::Null; + ser.Serialise("", present); + } + + const SDFile &file = ser.GetStructuredFile(); + + switch(el->type.basetype) + { + case SDBasic::Chunk: RDCERR("Unexpected chunk inside object!"); break; + case SDBasic::Struct: + for(size_t o = 0; o < el->data.children.size(); o++) + ser.Serialise("", el->data.children[o]); + break; + case SDBasic::Array: ser.Serialise("", (rdcarray &)el->data.children); break; + case SDBasic::Null: + // nothing to do, we serialised present flag above + RDCASSERT(el->type.flags & SDTypeFlags::Nullable); + break; + case SDBasic::Buffer: + { + size_t bufID = (size_t)el->data.basic.u; + byte *buf = file.buffers[bufID]->data(); + uint64_t size = file.buffers[bufID]->size(); + ser.Serialise("", buf, size); + break; + } + case SDBasic::String: + { + if(el->type.flags & SDTypeFlags::NullString) + { + const char *nullstring = NULL; + ser.Serialise("", nullstring); + } + else + { + ser.Serialise("", el->data.str); + } + break; + } + case SDBasic::Enum: + { + uint32_t e = (uint32_t)el->data.basic.u; + ser.Serialise("", e); + break; + } + case SDBasic::Boolean: ser.Serialise("", el->data.basic.b); break; + case SDBasic::Character: ser.Serialise("", el->data.basic.c); break; + case SDBasic::UnsignedInteger: + if(el->type.byteSize == 1) + { + uint8_t u = uint8_t(el->data.basic.u); + ser.Serialise("", u); + } + else if(el->type.byteSize == 2) + { + uint16_t u = uint16_t(el->data.basic.u); + ser.Serialise("", u); + } + else if(el->type.byteSize == 4) + { + uint32_t u = uint32_t(el->data.basic.u); + ser.Serialise("", u); + } + else if(el->type.byteSize == 8) + { + ser.Serialise("", el->data.basic.u); + } + else + { + RDCERR("Unexpeted integer size %u", el->type.byteSize); + } + break; + case SDBasic::SignedInteger: + if(el->type.byteSize == 1) + { + int8_t i = int8_t(el->data.basic.i); + ser.Serialise("", i); + } + else if(el->type.byteSize == 2) + { + int16_t i = int16_t(el->data.basic.i); + ser.Serialise("", i); + } + else if(el->type.byteSize == 4) + { + int32_t i = int32_t(el->data.basic.i); + ser.Serialise("", i); + } + else if(el->type.byteSize == 8) + { + ser.Serialise("", el->data.basic.i); + } + else + { + RDCERR("Unexpeted integer size %u", el->type.byteSize); + } + break; + case SDBasic::Float: + if(el->type.byteSize == 4) + { + float f = float(el->data.basic.d); + ser.Serialise("", f); + } + else if(el->type.byteSize == 8) + { + ser.Serialise("", el->data.basic.d); + } + else + { + RDCERR("Unexpeted float size %u", el->type.byteSize); + } + break; + } +} ///////////////////////////////////////////////////////////// // Basic types @@ -1946,11 +743,7 @@ std::string DoStringise(const int64_t &el) return StringFormat::Fmt("%lld", el); } -// this is super ugly, but I don't see a way around it - on other -// platforms size_t is typedef'd in such a way that the uint32_t or -// uint64_t specialisation will kick in. On apple, we need a -// specific size_t overload -#if ENABLED(RDOC_APPLE) +#if ENABLED(RDOC_SIZET_SEP_TYPE) template <> std::string DoStringise(const size_t &el) { @@ -1985,13 +778,13 @@ std::string DoStringise(const wchar_t &el) template <> std::string DoStringise(const byte &el) { - return StringFormat::Fmt("%08hhb", el); + return StringFormat::Fmt("%hhu", el); } template <> std::string DoStringise(const uint16_t &el) { - return StringFormat::Fmt("%04u", el); + return StringFormat::Fmt("%hu", el); } template <> @@ -2003,7 +796,7 @@ std::string DoStringise(const int32_t &el) template <> std::string DoStringise(const int16_t &el) { - return StringFormat::Fmt("%04d", el); + return StringFormat::Fmt("%hd", el); } template <> diff --git a/renderdoc/serialise/serialiser.h b/renderdoc/serialise/serialiser.h index 4c4f870d1..442527a74 100644 --- a/renderdoc/serialise/serialiser.h +++ b/renderdoc/serialise/serialiser.h @@ -1,8 +1,7 @@ /****************************************************************************** * The MIT License (MIT) * - * Copyright (c) 2015-2017 Baldur Karlsson - * Copyright (c) 2014 Crytek + * Copyright (c) 2017 Baldur Karlsson * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -25,40 +24,1620 @@ #pragma once -#include -#include #include #include #include -#include #include -#include "api/replay/basic_types.h" -#include "common/common.h" -#include "os/os_specific.h" +#include "api/replay/renderdoc_replay.h" +#include "streamio.h" -using std::set; -using std::string; +// function to deallocate anything from a serialise. Default impl +// does no deallocation of anything. +template +void Deserialise(const T &el) +{ +} -typedef const char *(*ChunkLookup)(uint32_t chunkType); +#define DECLARE_DESERIALISE_TYPE(type) \ + template <> \ + void Deserialise(const type &el); + +// this is a bit of a hack, but necessary. We don't want to have all of our DoSerialise functions +// defined in global cross-project headers and compiled everywhere, we just want to define them in +// .cpp files. +// However because they are template specializations, so they need to be explicitly instantiated in +// one file to link elsewhere. +#define INSTANTIATE_SERIALISE_TYPE(type) \ + template void DoSerialise(Serialiser &, type &); \ + template void DoSerialise(Serialiser &, type &); + +typedef std::string (*ChunkLookup)(uint32_t chunkType); + +enum class SerialiserFlags +{ + NoFlags = 0x0, + AllocateMemory = 0x1, +}; + +BITMASK_OPERATORS(SerialiserFlags); + +// This class is used to read and write arbitrary structured data from a stream. The primary +// mechanism is in template overloads of DoSerialise functions for each struct that can be +// serialised, down to primitive types (ints, floats, strings, etc). +// +// A serialised stream is defined in terms of 'chunks', each with an ID and some metadata attached +// like timestamp, duration, callstack, etc. Each chunk can then have objects serialised into it +// which automatically recurse and serialise all their members. When reading from/writing to a +// stream then the binary form is purely data - no structure encoded. When reading, the data can be +// optionally pulled out into a structured form that can then be manipulated or exported. +// +// Since a stream can be an in-memory buffer, a file, or a network socket this class is used to +// serialised complex data anywhere that we need structured I/O. + +enum class SerialiserMode +{ + Writing, + Reading, +}; -class Serialiser; -class ScopedContext; struct CompressedFileIO; +template +class Serialiser +{ +public: + static constexpr bool IsReading() { return sertype != SerialiserMode::Writing; } + static constexpr bool IsWriting() { return sertype == SerialiserMode::Writing; } + bool ExportStructure() const + { + return sertype == SerialiserMode::Reading && m_ExportStructured && !m_InternalElement; + } + + enum ChunkFlags + { + ChunkIndexMask = 0x0000ffff, + ChunkCallstack = 0x00010000, + ChunkThreadID = 0x00020000, + ChunkDuration = 0x00040000, + ChunkTimestamp = 0x00080000, + }; + + ////////////////////////////////////////// + // Init and error handling + ~Serialiser(); + + // no copies + Serialiser(const Serialiser &other) = delete; + + bool IsErrored() { return IsReading() ? m_Read->IsErrored() : m_Write->IsErrored(); } + StreamWriter *GetWriter() { return m_Write; } + StreamReader *GetReader() { return m_Read; } + void SetChunkMetadataRecording(uint32_t flags); + + SDChunkMetaData &ChunkMetadata() { return m_ChunkMetadata; } + ////////////////////////////////////////// + // Utility functions + + static uint64_t GetChunkAlignment() { return ChunkAlignment; } + void *GetUserData() { return m_pUserData; } + void SetUserData(void *userData) { m_pUserData = userData; } + void SetStringDatabase(std::set *db) { m_ExtStringDB = db; } + // jumps to the byte after the current chunk, can be called any time after BeginChunk + void SkipCurrentChunk(); + + // enable 'streaming mode' for ephemeral transfers like temporary I/O over sockets, where there's + // no need for the chunk length - avoids needing to seek internally in a stream that might not + // support seeking to fixup lengths, while also not requiring conservative length estimates + // up-front + void SetStreamingMode(bool stream) { m_DataStreaming = stream; } + SDFile &GetStructuredFile() { return *m_StructuredFile; } + void WriteStructuredFile(const SDFile &file); + void SetDrawChunk() { m_DrawChunk = true; } + ////////////////////////////////////////// + // Public serialisation interface + + void ConfigureStructuredExport(ChunkLookup lookup, bool includeBuffers) + { + m_ChunkLookup = lookup; + m_ExportBuffers = includeBuffers; + m_ExportStructured = (lookup != NULL); + } + + uint32_t BeginChunk(uint32_t chunkID, uint32_t byteLength); + void EndChunk(); + + ///////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////////// + // Templated serialisation functions + + // Users of the serialisation call Serialise() on the objects they want + // to serialise + + // for structure type members or external use, main entry point + + template + void SerialiseStringify(const T el) + { + if(ExportStructure()) + { + m_StructureStack.back()->data.str = ToStr(el); + m_StructureStack.back()->type.flags |= SDTypeFlags::HasCustomString; + } + } + + // serialise an object (either loose, or a structure member). + template + Serialiser &Serialise(const char *name, T &el, SerialiserFlags flags = SerialiserFlags::NoFlags) + { + if(ExportStructure()) + { + if(m_StructureStack.empty()) + { + RDCERR("Serialising object outside of chunk context! Start Chunk before any Serialise!"); + return *this; + } + + SDObject ¤t = *m_StructureStack.back(); + + current.data.basic.numChildren++; + current.data.children.push_back(new SDObject(name, TypeName())); + m_StructureStack.push_back(current.data.children.back()); + + SDObject &obj = *m_StructureStack.back(); + obj.type.byteSize = sizeof(T); + } + + SerialiseDispatch::Do(*this, el); + + if(ExportStructure()) + m_StructureStack.pop_back(); + + return *this; + } + + // special function for serialising buffers + Serialiser &Serialise(const char *name, byte *&el, uint64_t &byteSize, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + uint64_t count = byteSize; + + // silently handle NULL buffers + if(IsWriting() && el == NULL) + count = 0; + + { + m_InternalElement = true; + DoSerialise(*this, count); + m_InternalElement = false; + } + + if(IsReading()) + byteSize = count; + + if(ExportStructure()) + { + if(m_StructureStack.empty()) + { + RDCERR("Serialising object outside of chunk context! Start Chunk before any Serialise!"); + return *this; + } + + SDObject ¤t = *m_StructureStack.back(); + + current.data.basic.numChildren++; + current.data.children.push_back(new SDObject(name, "Byte Buffer")); + m_StructureStack.push_back(current.data.children.back()); + + SDObject &obj = *m_StructureStack.back(); + obj.type.basetype = SDBasic::Buffer; + obj.type.byteSize = byteSize; + } + + { + if(IsWriting()) + { + // ensure byte alignment + m_Write->AlignTo(); + + if(el) + m_Write->Write(el, byteSize); + else + RDCASSERT(byteSize == 0); + } + else if(IsReading()) + { + // ensure byte alignment + m_Read->AlignTo(); + +// Coverity is unable to tie this allocation together with the automatic scoped deallocation in the +// ScopedDeseralise* classes. We can verify with e.g. valgrind that there are no leaks, so to keep +// the analysis non-spammy we just don't allocate for coverity builds +#if !defined(__COVERITY__) + if(flags & SerialiserFlags::AllocateMemory) + { + if(byteSize > 0) + el = AllocAlignedBuffer(byteSize); + else + el = NULL; + } +#endif + + m_Read->Read(el, byteSize); + } + } + + if(ExportStructure()) + { + if(m_ExportBuffers) + { + SDObject &obj = *m_StructureStack.back(); + + obj.data.basic.u = m_StructuredFile->buffers.size(); + + bytebuf *alloc = new bytebuf; + alloc->resize((size_t)byteSize); + if(el) + memcpy(alloc->data(), el, (size_t)byteSize); + + m_StructuredFile->buffers.push_back(alloc); + } + + m_StructureStack.pop_back(); + } + + return *this; + } + + Serialiser &Serialise(const char *name, byte *&el, uint32_t &byteSize, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + uint64_t upcastSize = byteSize; + Serialise(name, el, upcastSize, flags); + if(IsReading()) + byteSize = (uint32_t)upcastSize; + return *this; + } + +#if ENABLED(RDOC_SIZET_SEP_TYPE) + Serialiser &Serialise(const char *name, byte *&el, size_t &byteSize, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + uint64_t upcastSize = (uint64_t)byteSize; + Serialise(name, el, upcastSize, flags); + if(IsReading()) + byteSize = (size_t)upcastSize; + return *this; + } +#endif + + Serialiser &Serialise(const char *name, bytebuf &el, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + uint64_t count = (uint64_t)el.size(); + + { + m_InternalElement = true; + DoSerialise(*this, count); + m_InternalElement = false; + } + + if(ExportStructure()) + { + if(m_StructureStack.empty()) + { + RDCERR("Serialising object outside of chunk context! Start Chunk before any Serialise!"); + return *this; + } + + SDObject ¤t = *m_StructureStack.back(); + + current.data.basic.numChildren++; + current.data.children.push_back(new SDObject(name, "Byte Buffer")); + m_StructureStack.push_back(current.data.children.back()); + + SDObject &obj = *m_StructureStack.back(); + obj.type.basetype = SDBasic::Buffer; + obj.type.byteSize = count; + } + + { + if(IsWriting()) + { + // ensure byte alignment + m_Write->AlignTo(); + m_Write->Write(el.data(), count); + } + else if(IsReading()) + { + // ensure byte alignment + m_Read->AlignTo(); + + el.resize((size_t)count); + + m_Read->Read(el.data(), count); + } + } + + if(ExportStructure()) + { + if(m_ExportBuffers) + { + SDObject &obj = *m_StructureStack.back(); + + obj.data.basic.u = m_StructuredFile->buffers.size(); + + bytebuf *alloc = new bytebuf; + alloc->assign(el); + + m_StructuredFile->buffers.push_back(alloc); + } + + m_StructureStack.pop_back(); + } + + return *this; + } + + Serialiser &Serialise(const char *name, std::vector &el, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + uint64_t count = (uint64_t)el.size(); + + { + m_InternalElement = true; + DoSerialise(*this, count); + m_InternalElement = false; + } + + if(ExportStructure()) + { + if(m_StructureStack.empty()) + { + RDCERR("Serialising object outside of chunk context! Start Chunk before any Serialise!"); + return *this; + } + + SDObject ¤t = *m_StructureStack.back(); + + current.data.basic.numChildren++; + current.data.children.push_back(new SDObject(name, "Byte Buffer")); + m_StructureStack.push_back(current.data.children.back()); + + SDObject &obj = *m_StructureStack.back(); + obj.type.basetype = SDBasic::Buffer; + obj.type.byteSize = count; + } + + { + if(IsWriting()) + { + // ensure byte alignment + m_Write->AlignTo(); + m_Write->Write(el.data(), count); + } + else if(IsReading()) + { + // ensure byte alignment + m_Read->AlignTo(); + + el.resize((size_t)count); + + m_Read->Read(el.data(), count); + } + } + + if(ExportStructure()) + { + if(m_ExportBuffers) + { + SDObject &obj = *m_StructureStack.back(); + + obj.data.basic.u = m_StructuredFile->buffers.size(); + + bytebuf *alloc = new bytebuf; + alloc->resize((size_t)count); + memcpy(alloc->data(), el.data(), alloc->size()); + + m_StructuredFile->buffers.push_back(alloc); + } + + m_StructureStack.pop_back(); + } + + return *this; + } + + Serialiser &Serialise(const char *name, void *&el, uint64_t &byteSize, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + return Serialise(name, (byte *&)el, byteSize, flags); + } + + Serialiser &Serialise(const char *name, const void *&el, uint64_t &byteSize, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + return Serialise(name, (byte *&)el, byteSize, flags); + } + + Serialiser &Serialise(const char *name, void *&el, uint32_t &byteSize, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + return Serialise(name, (byte *&)el, byteSize, flags); + } + + Serialiser &Serialise(const char *name, const void *&el, uint32_t &byteSize, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + return Serialise(name, (byte *&)el, byteSize, flags); + } + +#if ENABLED(RDOC_SIZET_SEP_TYPE) + Serialiser &Serialise(const char *name, void *&el, size_t &byteSize, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + return Serialise(name, (byte *&)el, byteSize, flags); + } + + Serialiser &Serialise(const char *name, const void *&el, size_t &byteSize, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + return Serialise(name, (byte *&)el, byteSize, flags); + } +#endif + +#if ENABLED(RDOC_WIN32) + // annoyingly, windows SIZE_T is unsigned long on win32 which is a different type to + // uint32_t/uint64_t. So we add a special overload here for its sake. + Serialiser &Serialise(const char *name, const void *&el, unsigned long &byteSize, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + uint64_t bs = byteSize; + Serialiser &ret = Serialise(name, (byte *&)el, bs, flags); + byteSize = (unsigned long)bs; + return ret; + } +#endif + + // serialise a fixed array like foo[4]; + // never needs to allocate, just needs to be iterated + template + Serialiser &Serialise(const char *name, T (&el)[N], SerialiserFlags flags = SerialiserFlags::NoFlags) + { + // for consistency with other arrays, even though this is redundant, we serialise out and in the + // size + uint64_t count = N; + { + m_InternalElement = true; + DoSerialise(*this, count); + m_InternalElement = false; + + if(count != N) + RDCWARN("Fixed-size array length %zu serialised with different size %llu", N, count); + } + + if(ExportStructure()) + { + if(m_StructureStack.empty()) + { + RDCERR("Serialising object outside of chunk context! Start Chunk before any Serialise!"); + return *this; + } + + SDObject &parent = *m_StructureStack.back(); + parent.data.basic.numChildren++; + parent.data.children.push_back(new SDObject(name, TypeName())); + m_StructureStack.push_back(parent.data.children.back()); + + SDObject &arr = *m_StructureStack.back(); + arr.type.basetype = SDBasic::Array; + arr.type.byteSize = N; + + arr.data.basic.numChildren = (uint64_t)N; + arr.data.children.resize(N); + + for(size_t i = 0; i < N; i++) + { + arr.data.children[i] = new SDObject("$el", TypeName()); + m_StructureStack.push_back(arr.data.children[i]); + + SDObject &obj = *m_StructureStack.back(); + + // default to struct. This will be overwritten if appropriate + obj.type.basetype = SDBasic::Struct; + obj.type.byteSize = sizeof(T); + + // Check against the serialised count here - on read if we don't have the right size this + // means we won't read past the provided data. + if(i < count) + { + SerialiseDispatch::Do(*this, el[i]); + } + else + { + // we should have data for these elements, but we don't. Just default initialise + el[i] = T(); + } + + m_StructureStack.pop_back(); + } + + // if we have more data than the fixed sized array allows, we must simply discard the excess + if(count > N) + { + // prevent any trashing of structured data by these + bool wasInternal = m_InternalElement; + m_InternalElement = true; + T dummy; + SerialiseDispatch::Do(*this, dummy); + m_InternalElement = wasInternal; + } + + m_StructureStack.pop_back(); + } + else + { + for(size_t i = 0; i < N && i < count; i++) + SerialiseDispatch::Do(*this, el[i]); + + for(size_t i = N; i < count; i++) + { + T dummy; + SerialiseDispatch::Do(*this, dummy); + } + } + + return *this; + } + + // specialisation for fixed character arrays, serialising as strings + template + Serialiser &Serialise(const char *name, char (&el)[N], + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + std::string str = el; + Serialise(name, str, flags); + if(str.length() >= N) + { + RDCWARN("Serialising string too large for fixed-size array '%s', will be truncated", name); + memcpy(el, str.c_str(), N - 1); + el[N - 1] = 0; + } + else + { + // copy the string & trailing NULL into el. + memcpy(el, str.c_str(), str.length() + 1); + } + + return *this; + } + + // special function for serialising dynamically sized arrays + template + Serialiser &Serialise(const char *name, T *&el, uint64_t &arrayCount, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + uint64_t count = arrayCount; + + // silently handle NULL arrays + if(IsWriting() && el == NULL) + count = 0; + + { + m_InternalElement = true; + DoSerialise(*this, count); + m_InternalElement = false; + } + + if(IsReading()) + arrayCount = count; + + if(ExportStructure()) + { + if(m_StructureStack.empty()) + { + RDCERR("Serialising object outside of chunk context! Start Chunk before any Serialise!"); + return *this; + } + + SDObject &parent = *m_StructureStack.back(); + parent.data.basic.numChildren++; + parent.data.children.push_back(new SDObject(name, TypeName())); + m_StructureStack.push_back(parent.data.children.back()); + + SDObject &arr = *m_StructureStack.back(); + arr.type.basetype = SDBasic::Array; + arr.type.byteSize = arrayCount; + + arr.data.basic.numChildren = arrayCount; + arr.data.children.resize((size_t)arrayCount); + +// Coverity is unable to tie this allocation together with the automatic scoped deallocation in the +// ScopedDeseralise* classes. We can verify with e.g. valgrind that there are no leaks, so to keep +// the analysis non-spammy we just don't allocate for coverity builds +#if !defined(__COVERITY__) + if(IsReading() && (flags & SerialiserFlags::AllocateMemory)) + { + if(arrayCount > 0) + el = new T[(size_t)arrayCount]; + else + el = NULL; + } +#endif + + for(uint64_t i = 0; el && i < arrayCount; i++) + { + arr.data.children[(size_t)i] = new SDObject("$el", TypeName()); + m_StructureStack.push_back(arr.data.children[(size_t)i]); + + SDObject &obj = *m_StructureStack.back(); + + // default to struct. This will be overwritten if appropriate + obj.type.basetype = SDBasic::Struct; + obj.type.byteSize = sizeof(T); + + SerialiseDispatch::Do(*this, el[i]); + + m_StructureStack.pop_back(); + } + + m_StructureStack.pop_back(); + } + else + { +// Coverity is unable to tie this allocation together with the automatic scoped deallocation in the +// ScopedDeseralise* classes. We can verify with e.g. valgrind that there are no leaks, so to keep +// the analysis non-spammy we just don't allocate for coverity builds +#if !defined(__COVERITY__) + if(IsReading() && (flags & SerialiserFlags::AllocateMemory)) + { + if(arrayCount > 0) + el = new T[(size_t)arrayCount]; + else + el = NULL; + } +#endif + + for(size_t i = 0; el && i < arrayCount; i++) + SerialiseDispatch::Do(*this, el[i]); + } + + return *this; + } + + template + Serialiser &Serialise(const char *name, T *&el, uint32_t &arrayCount, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + uint64_t upcastCount = arrayCount; + Serialise(name, el, upcastCount, flags); + if(IsReading()) + arrayCount = (uint32_t)upcastCount; + return *this; + } + + // specialisations for container types + template + Serialiser &Serialise(const char *name, std::vector &el, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + uint64_t size = (uint64_t)el.size(); + + { + m_InternalElement = true; + DoSerialise(*this, size); + m_InternalElement = false; + } + + if(ExportStructure()) + { + if(m_StructureStack.empty()) + { + RDCERR("Serialising object outside of chunk context! Start Chunk before any Serialise!"); + return *this; + } + + SDObject &parent = *m_StructureStack.back(); + parent.data.basic.numChildren++; + parent.data.children.push_back(new SDObject(name, TypeName())); + m_StructureStack.push_back(parent.data.children.back()); + + SDObject &arr = *m_StructureStack.back(); + arr.type.basetype = SDBasic::Array; + arr.type.byteSize = size; + + arr.data.basic.numChildren = size; + arr.data.children.resize((size_t)size); + + if(IsReading()) + el.resize((size_t)size); + + for(size_t i = 0; i < (size_t)size; i++) + { + arr.data.children[i] = new SDObject("$el", TypeName()); + m_StructureStack.push_back(arr.data.children[i]); + + SDObject &obj = *m_StructureStack.back(); + + // default to struct. This will be overwritten if appropriate + obj.type.basetype = SDBasic::Struct; + obj.type.byteSize = sizeof(U); + + SerialiseDispatch::Do(*this, el[i]); + + m_StructureStack.pop_back(); + } + + m_StructureStack.pop_back(); + } + else + { + if(IsReading()) + el.resize((size_t)size); + + for(size_t i = 0; i < (size_t)size; i++) + SerialiseDispatch::Do(*this, el[i]); + } + + return *this; + } + + template + Serialiser &Serialise(const char *name, std::list &el, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + uint64_t size = (uint64_t)el.size(); + + { + m_InternalElement = true; + DoSerialise(*this, size); + m_InternalElement = false; + } + + if(ExportStructure()) + { + if(m_StructureStack.empty()) + { + RDCERR("Serialising object outside of chunk context! Start Chunk before any Serialise!"); + return *this; + } + + SDObject &parent = *m_StructureStack.back(); + parent.data.basic.numChildren++; + parent.data.children.push_back(new SDObject(name, TypeName())); + m_StructureStack.push_back(parent.data.children.back()); + + SDObject &arr = *m_StructureStack.back(); + arr.type.basetype = SDBasic::Array; + arr.type.byteSize = size; + + arr.data.basic.numChildren = size; + arr.data.children.resize((size_t)size); + + if(IsReading()) + el.resize((size_t)size); + + auto it = el.begin(); + + for(size_t i = 0; i < (size_t)size; i++) + { + arr.data.children[i] = new SDObject("$el", TypeName()); + m_StructureStack.push_back(arr.data.children[i]); + + SDObject &obj = *m_StructureStack.back(); + + // default to struct. This will be overwritten if appropriate + obj.type.basetype = SDBasic::Struct; + obj.type.byteSize = sizeof(U); + + SerialiseDispatch::Do(*this, *it); + it++; + + m_StructureStack.pop_back(); + } + + m_StructureStack.pop_back(); + } + else + { + if(IsReading()) + el.resize((size_t)size); + + for(auto it = el.begin(); it != el.end(); ++it) + SerialiseDispatch::Do(*this, *it); + } + + return *this; + } + + template + Serialiser &Serialise(const char *name, std::pair &el, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + if(ExportStructure()) + { + if(m_StructureStack.empty()) + { + RDCERR("Serialising object outside of chunk context! Start Chunk before any Serialise!"); + return *this; + } + + SDObject &parent = *m_StructureStack.back(); + parent.data.basic.numChildren++; + parent.data.children.push_back(new SDObject(name, "pair")); + m_StructureStack.push_back(parent.data.children.back()); + + SDObject &arr = *m_StructureStack.back(); + arr.type.basetype = SDBasic::Struct; + arr.type.byteSize = 2; + + arr.data.basic.numChildren = 2; + arr.data.children.resize(2); + + { + arr.data.children[0] = new SDObject("first", TypeName()); + m_StructureStack.push_back(arr.data.children[0]); + + SDObject &obj = *m_StructureStack.back(); + + // default to struct. This will be overwritten if appropriate + obj.type.basetype = SDBasic::Struct; + obj.type.byteSize = sizeof(U); + + SerialiseDispatch::Do(*this, el.first); + + m_StructureStack.pop_back(); + } + + { + arr.data.children[1] = new SDObject("second", TypeName()); + m_StructureStack.push_back(arr.data.children[1]); + + SDObject &obj = *m_StructureStack.back(); + + // default to struct. This will be overwritten if appropriate + obj.type.basetype = SDBasic::Struct; + obj.type.byteSize = sizeof(V); + + SerialiseDispatch::Do(*this, el.second); + + m_StructureStack.pop_back(); + } + + m_StructureStack.pop_back(); + } + else + { + SerialiseDispatch::Do(*this, el.first); + SerialiseDispatch::Do(*this, el.second); + } + + return *this; + } + + template + Serialiser &Serialise(const char *name, rdcarray &el, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + uint64_t size = (uint64_t)el.size(); + + { + m_InternalElement = true; + DoSerialise(*this, size); + m_InternalElement = false; + } + + if(ExportStructure()) + { + if(m_StructureStack.empty()) + { + RDCERR("Serialising object outside of chunk context! Start Chunk before any Serialise!"); + return *this; + } + + SDObject &parent = *m_StructureStack.back(); + parent.data.basic.numChildren++; + parent.data.children.push_back(new SDObject(name, TypeName())); + m_StructureStack.push_back(parent.data.children.back()); + + SDObject &arr = *m_StructureStack.back(); + arr.type.basetype = SDBasic::Array; + arr.type.byteSize = size; + + arr.data.basic.numChildren = size; + arr.data.children.resize((size_t)size); + + if(IsReading()) + el.resize((int)size); + + for(size_t i = 0; i < (size_t)size; i++) + { + arr.data.children[i] = new SDObject("$el", TypeName()); + m_StructureStack.push_back(arr.data.children[i]); + + SDObject &obj = *m_StructureStack.back(); + + // default to struct. This will be overwritten if appropriate + obj.type.basetype = SDBasic::Struct; + obj.type.byteSize = sizeof(U); + + SerialiseDispatch::Do(*this, el[i]); + + m_StructureStack.pop_back(); + } + + m_StructureStack.pop_back(); + } + else + { + if(IsReading()) + el.resize((int)size); + + for(size_t i = 0; i < (size_t)size; i++) + SerialiseDispatch::Do(*this, el[i]); + } + + return *this; + } + + template + Serialiser &Serialise(const char *name, rdcpair &el, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + if(ExportStructure()) + { + if(m_StructureStack.empty()) + { + RDCERR("Serialising object outside of chunk context! Start Chunk before any Serialise!"); + return *this; + } + + SDObject &parent = *m_StructureStack.back(); + parent.data.basic.numChildren++; + parent.data.children.push_back(new SDObject(name, "pair")); + m_StructureStack.push_back(parent.data.children.back()); + + SDObject &arr = *m_StructureStack.back(); + arr.type.basetype = SDBasic::Struct; + arr.type.byteSize = 2; + + arr.data.basic.numChildren = 2; + arr.data.children.resize(2); + + { + arr.data.children[0] = new SDObject("first", TypeName()); + m_StructureStack.push_back(arr.data.children[0]); + + SDObject &obj = *m_StructureStack.back(); + + // default to struct. This will be overwritten if appropriate + obj.type.basetype = SDBasic::Struct; + obj.type.byteSize = sizeof(U); + + SerialiseDispatch::Do(*this, el.first); + + m_StructureStack.pop_back(); + } + + { + arr.data.children[1] = new SDObject("second", TypeName()); + m_StructureStack.push_back(arr.data.children[1]); + + SDObject &obj = *m_StructureStack.back(); + + // default to struct. This will be overwritten if appropriate + obj.type.basetype = SDBasic::Struct; + obj.type.byteSize = sizeof(V); + + SerialiseDispatch::Do(*this, el.second); + + m_StructureStack.pop_back(); + } + + m_StructureStack.pop_back(); + } + else + { + SerialiseDispatch::Do(*this, el.first); + SerialiseDispatch::Do(*this, el.second); + } + + return *this; + } + + // for const types, to cast away the const! + // caller is responsible for ensuring that on write, it's safe to write into + // this pointer on the other end + template + Serialiser &Serialise(const char *name, const T &el, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + return Serialise(name, (T &)el, flags); + } + + // dynamic array variant + template + Serialiser &Serialise(const char *name, const T *&el, uint64_t &arrayCount, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + return Serialise(name, (T *&)el, arrayCount, flags); + } + + template + Serialiser &SerialiseNullable(const char *name, T *&el, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + bool present = (el != NULL); + + { + m_InternalElement = true; + DoSerialise(*this, present); + m_InternalElement = false; + } + + if(ExportStructure()) + { + if(m_StructureStack.empty()) + { + RDCERR("Serialising object outside of chunk context! Start Chunk before any Serialise!"); + return *this; + } + +// Coverity is unable to tie this allocation together with the automatic scoped deallocation in the +// ScopedDeseralise* classes. We can verify with e.g. valgrind that there are no leaks, so to keep +// the analysis non-spammy we just don't allocate for coverity builds +#if !defined(__COVERITY__) + if(IsReading()) + { + if(present) + el = new T; + else + el = NULL; + } +#endif + + if(el) + { + Serialise(name, *el, flags); + + SDObject &parent = *m_StructureStack.back(); + + SDObject &nullable = *parent.data.children.back(); + + nullable.type.flags |= SDTypeFlags::Nullable; + } + else + { + SDObject &parent = *m_StructureStack.back(); + parent.data.basic.numChildren++; + parent.data.children.push_back(new SDObject(name, TypeName())); + + SDObject &nullable = *parent.data.children.back(); + nullable.type.basetype = SDBasic::Null; + nullable.type.byteSize = 0; + nullable.type.flags |= SDTypeFlags::Nullable; + } + } + else + { +// Coverity is unable to tie this allocation together with the automatic scoped deallocation in the +// ScopedDeseralise* classes. We can verify with e.g. valgrind that there are no leaks, so to keep +// the analysis non-spammy we just don't allocate for coverity builds +#if !defined(__COVERITY__) + if(IsReading()) + { + if(present) + el = new T; + else + el = NULL; + } +#endif + + if(el) + Serialise(name, *el, flags); + } + + return *this; + } + + template + Serialiser &SerialiseNullable(const char *name, const T *&el, + SerialiserFlags flags = SerialiserFlags::NoFlags) + { + return SerialiseNullable(name, (T *&)el, flags); + } + + Serialiser &SerialiseStream(const std::string &name, StreamReader &stream, float *progress = NULL) + { + RDCCOMPILE_ASSERT(IsWriting(), "Can't read into a StreamReader"); + + uint64_t totalSize = stream.GetSize(); + + { + m_InternalElement = true; + DoSerialise(*this, totalSize); + m_InternalElement = false; + } + + // ensure byte alignment + m_Write->AlignTo(); + + StreamTransfer(m_Write, &stream, progress); + + return *this; + } + + Serialiser &SerialiseStream(const std::string &name, StreamWriter &stream, float *progress = NULL) + { + RDCCOMPILE_ASSERT(IsReading(), "Can't write from a StreamWriter"); + + uint64_t totalSize = 0; + + { + m_InternalElement = true; + DoSerialise(*this, totalSize); + m_InternalElement = false; + } + + size_t byteSize = (size_t)totalSize; + + byte *structBuf = NULL; + + if(ExportStructure()) + { + if(m_StructureStack.empty()) + { + RDCERR("Serialising object outside of chunk context! Start Chunk before any Serialise!"); + return *this; + } + + SDObject ¤t = *m_StructureStack.back(); + + current.data.basic.numChildren++; + current.data.children.push_back(new SDObject(name.c_str(), "Byte Buffer")); + m_StructureStack.push_back(current.data.children.back()); + + SDObject &obj = *m_StructureStack.back(); + obj.type.basetype = SDBasic::Buffer; + obj.type.byteSize = totalSize; + + if(m_ExportBuffers) + { + obj.data.basic.u = m_StructuredFile->buffers.size(); + + m_StructuredFile->buffers.push_back(new bytebuf); + m_StructuredFile->buffers.back()->resize((size_t)totalSize); + + // this will be filled as we read below + structBuf = m_StructuredFile->buffers.back()->data(); + } + + m_StructureStack.pop_back(); + } + + // ensure byte alignment + m_Read->AlignTo(); + + if(totalSize > 0) + { + // copy 1MB at a time + const uint64_t StreamIOChunkSize = 1024 * 1024; + + // copy 1MB at a time + const uint64_t bufSize = RDCMIN(StreamIOChunkSize, totalSize); + uint64_t numBufs = totalSize / bufSize; + // last remaining partial buffer + if(totalSize % (uint64_t)bufSize > 0) + numBufs++; + + byte *buf = new byte[byteSize]; + + if(progress) + *progress = 0.0001f; + + for(uint64_t i = 0; i < numBufs; i++) + { + uint64_t payloadLength = RDCMIN(bufSize, totalSize); + + m_Read->Read(buf, payloadLength); + stream.Write(buf, payloadLength); + + if(structBuf) + { + memcpy(structBuf, buf, (size_t)payloadLength); + structBuf += payloadLength; + } + + totalSize -= payloadLength; + if(progress) + *progress = float(i + 1) / float(numBufs); + } + + delete[] buf; + } + else + { + if(progress) + *progress = 1.0f; + } + + return *this; + } + + // these functions can be chained onto the end of a Serialise() call or macro to + // set additional properties or change things + Serialiser &Hidden() + { + if(ExportStructure() && !m_StructureStack.empty()) + { + SDObject ¤t = *m_StructureStack.back(); + + if(!current.data.children.empty()) + current.data.children.back()->type.flags |= SDTypeFlags::Hidden; + } + + return *this; + } + + Serialiser &Named(const char *name) + { + if(ExportStructure() && !m_StructureStack.empty()) + { + SDObject ¤t = *m_StructureStack.back(); + + if(!current.data.children.empty()) + current.data.children.back()->name = name; + } + + return *this; + } + + ///////////////////////////////////////////////////////////////////////////// + + // for basic/leaf types. Read/written just as byte soup, MUST be plain old data + template + void SerialiseValue(SDBasic type, size_t byteSize, T &el) + { + if(IsWriting()) + { + m_Write->Write(el); + } + else if(IsReading()) + { + m_Read->Read(el); + } + + if(!ExportStructure()) + return; + + SDObject ¤t = *m_StructureStack.back(); + + current.type.basetype = type; + current.type.byteSize = byteSize; + + switch(type) + { + case SDBasic::Chunk: + case SDBasic::Struct: + case SDBasic::Array: + case SDBasic::Buffer: + case SDBasic::Null: RDCFATAL("Cannot call SerialiseValue for type %d!", type); break; + case SDBasic::String: RDCFATAL("eString should be specialised!"); break; + case SDBasic::Enum: + case SDBasic::UnsignedInteger: + if(byteSize == 1) + current.data.basic.u = (uint64_t)(uint8_t)el; + else if(byteSize == 2) + current.data.basic.u = (uint64_t)(uint16_t)el; + else if(byteSize == 4) + current.data.basic.u = (uint64_t)(uint32_t)el; + else if(byteSize == 8) + current.data.basic.u = (uint64_t)el; + else + RDCFATAL("Unsupported unsigned integer byte width: %u", byteSize); + break; + case SDBasic::SignedInteger: + if(byteSize == 1) + current.data.basic.i = (int64_t)(int8_t)el; + else if(byteSize == 2) + current.data.basic.i = (int64_t)(int16_t)el; + else if(byteSize == 4) + current.data.basic.i = (int64_t)(int32_t)el; + else if(byteSize == 8) + current.data.basic.i = (int64_t)el; + else + RDCFATAL("Unsupported signed integer byte width: %u", byteSize); + break; + case SDBasic::Float: + if(byteSize == 4) + current.data.basic.d = (double)(float)el; + else if(byteSize == 8) + current.data.basic.d = (double)el; + else + RDCFATAL("Unsupported floating point byte width: %u", byteSize); + break; + case SDBasic::Boolean: + // co-erce to boolean + current.data.basic.b = !!(el); + break; + case SDBasic::Character: current.data.basic.c = (char)(el); break; + } + } + + // the only non POD basic type + void SerialiseValue(SDBasic type, size_t byteSize, std::string &el) + { + uint32_t len = 0; + + if(IsReading()) + { + m_Read->Read(len); + el.resize(len); + if(len > 0) + m_Read->Read(&el[0], len); + } + else + { + len = (uint32_t)el.length(); + m_Write->Write(len); + m_Write->Write(el.c_str(), len); + } + + if(ExportStructure()) + { + SDObject ¤t = *m_StructureStack.back(); + + current.type.basetype = type; + current.type.byteSize = len; + current.data.str = el; + } + } + + void SerialiseValue(SDBasic type, size_t byteSize, rdcstr &el) + { + uint32_t len = 0; + + if(IsReading()) + { + m_Read->Read(len); + el.resize((int)len); + if(len > 0) + m_Read->Read(&el[0], len); + } + else + { + len = (uint32_t)el.size(); + m_Write->Write(len); + m_Write->Write(el.c_str(), len); + } + + if(ExportStructure()) + { + SDObject ¤t = *m_StructureStack.back(); + + current.type.basetype = type; + current.type.byteSize = len; + current.data.str = el; + } + } + + void SerialiseValue(SDBasic type, size_t byteSize, char *&el) + { + int32_t len = 0; + + if(IsReading()) + { + m_Read->Read(len); + if(len == -1) + { + el = NULL; + } + else + { + std::string str; + str.resize(len); + if(len > 0) + m_Read->Read(&str[0], len); + el = (char *)StringDB(str); + } + } + else + { + len = el ? (int32_t)strlen(el) : -1; + m_Write->Write(len); + if(len > 0) + m_Write->Write(el, len); + } + + if(ExportStructure()) + { + SDObject ¤t = *m_StructureStack.back(); + + current.type.basetype = type; + current.type.byteSize = RDCMAX(len, 0); + current.data.str = el ? el : ""; + if(len == -1) + current.type.flags |= SDTypeFlags::NullString; + } + } + + void SerialiseValue(SDBasic type, size_t byteSize, const char *&el) + { + SerialiseValue(type, byteSize, (char *&)el); + } + + // constructors only available by the derived classes for each serialiser type +protected: + Serialiser(StreamWriter *writer, Ownership own); + Serialiser(StreamReader *reader, Ownership own); + +private: + static const uint64_t ChunkAlignment = 64; + template ::value> + struct SerialiseDispatch + { + static void Do(SerialiserMode &ser, T &el) { DoSerialise(ser, el); } + }; + + template + struct SerialiseDispatch + { + static void Do(SerialiserMode &ser, T &el) + { + typedef typename std::underlying_type::type etype; + constexpr bool is_valid_type = + std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value || + std::is_same::value; + RDCCOMPILE_ASSERT(is_valid_type, "enum isn't expected type"); + ser.SerialiseValue(SDBasic::Enum, sizeof(T), (etype &)(el)); + ser.SerialiseStringify(el); + } + }; + + void *m_pUserData = NULL; + + StreamWriter *m_Write = NULL; + StreamReader *m_Read = NULL; + + Ownership m_Ownership; + + // See SetStreamingMode + bool m_DataStreaming = false; + bool m_DrawChunk = false; + + uint64_t m_LastChunkOffset = 0; + uint64_t m_ChunkFixup = 0; + + bool m_ExportStructured = false; + bool m_ExportBuffers = false; + bool m_InternalElement = false; + SDFile m_StructData; + SDFile *m_StructuredFile = &m_StructData; + std::vector m_StructureStack; + + uint32_t m_ChunkFlags = 0; + SDChunkMetaData m_ChunkMetadata; + + // a database of strings read from the file, useful when serialised structures + // expect a char* to return and point to static memory + std::set m_StringDB; + + // external storage - so the string storage can persist after the lifetime of the serialiser + std::set *m_ExtStringDB = NULL; + + const char *StringDB(const std::string &s) + { + if(m_ExtStringDB) + { + auto it = m_ExtStringDB->insert(s); + return it.first->c_str(); + } + + auto it = m_StringDB.insert(s); + return it.first->c_str(); + } + + ChunkLookup m_ChunkLookup = NULL; +}; + +#ifndef SERIALISER_IMPL +class WriteSerialiser : public Serialiser +{ +public: + WriteSerialiser(StreamWriter *writer, Ownership own) : Serialiser(writer, own) {} + void WriteChunk(uint32_t chunkID, uint32_t byteLength = 0) { BeginChunk(chunkID, byteLength); } +}; + +class ReadSerialiser : public Serialiser +{ +public: + ReadSerialiser(StreamReader *reader, Ownership own) : Serialiser(reader, own) {} + template + ChunkType ReadChunk() + { + // parameters are ignored when reading + return (ChunkType)BeginChunk(0, 0); + } +}; +#endif + +#define BASIC_TYPE_SERIALISE(typeName, member, type, byteSize) \ + DECLARE_STRINGISE_TYPE(typeName) \ + template \ + void DoSerialise(SerialiserType &ser, typeName &el) \ + { \ + ser.SerialiseValue(type, byteSize, member); \ + } + +#define BASIC_TYPE_SERIALISE_STRINGIFY(typeName, member, type, byteSize) \ + template \ + void DoSerialise(SerialiserType &ser, typeName &el) \ + { \ + ser.SerialiseValue(type, byteSize, member); \ + ser.SerialiseStringify(el); \ + } + +BASIC_TYPE_SERIALISE(int64_t, el, SDBasic::SignedInteger, 8); +BASIC_TYPE_SERIALISE(uint64_t, el, SDBasic::UnsignedInteger, 8); +BASIC_TYPE_SERIALISE(int32_t, el, SDBasic::SignedInteger, 4); +BASIC_TYPE_SERIALISE(uint32_t, el, SDBasic::UnsignedInteger, 4); +BASIC_TYPE_SERIALISE(int16_t, el, SDBasic::SignedInteger, 2); +BASIC_TYPE_SERIALISE(uint16_t, el, SDBasic::UnsignedInteger, 2); +BASIC_TYPE_SERIALISE(int8_t, el, SDBasic::SignedInteger, 1); +BASIC_TYPE_SERIALISE(uint8_t, el, SDBasic::UnsignedInteger, 1); + +BASIC_TYPE_SERIALISE(double, el, SDBasic::Float, 8); +BASIC_TYPE_SERIALISE(float, el, SDBasic::Float, 4); + +BASIC_TYPE_SERIALISE(bool, el, SDBasic::Boolean, 1); + +BASIC_TYPE_SERIALISE(char, el, SDBasic::Character, 1); + +BASIC_TYPE_SERIALISE(char *, el, SDBasic::String, 0); +BASIC_TYPE_SERIALISE(const char *, el, SDBasic::String, 0); + +// these are special because we give them a typename of 'string' for both: +// BASIC_TYPE_SERIALISE(rdcstr, el, SDBasic::String, 0); +// BASIC_TYPE_SERIALISE(std::string, el, SDBasic::String, 0); + +template <> +inline const char *TypeName() +{ + return "string"; +} +template +void DoSerialise(SerialiserType &ser, std::string &el) +{ + ser.SerialiseValue(SDBasic::String, 0, el); +} +template <> +inline const char *TypeName() +{ + return "string"; +} +template +void DoSerialise(SerialiserType &ser, rdcstr &el) +{ + ser.SerialiseValue(SDBasic::String, 0, el); +} + +DECLARE_STRINGISE_TYPE(SDObject *); + +class ScopedChunk; + // holds the memory, length and type for a given chunk, so that it can be // passed around and moved between owners before being serialised out class Chunk { public: - ~Chunk(); - - const char *GetDebugString() { return m_DebugStr.c_str(); } - byte *GetData() { return m_Data; } - uint32_t GetLength() { return m_Length; } - uint32_t GetChunkType() { return m_ChunkType; } - bool IsAligned() { return m_AlignedData; } - bool IsTemporary() { return m_Temporary; } -#if ENABLED(RDOC_DEVEL) + ~Chunk() { FreeAlignedBuffer(m_Data); } + template + ChunkType GetChunkType() + { + return (ChunkType)m_ChunkType; + } +#if !defined(RELEASE) static uint64_t NumLiveChunks() { return m_LiveChunks; } static uint64_t TotalMem() { return m_TotalMem; } #else @@ -67,786 +1646,243 @@ public: #endif // grab current contents of the serialiser into this chunk - Chunk(Serialiser *ser, uint32_t chunkType, bool temp); + Chunk(Serialiser &ser, uint32_t chunkType) + { + m_Length = (uint32_t)ser.GetWriter()->GetOffset(); - Chunk *Duplicate(); + RDCASSERT(ser.GetWriter()->GetOffset() < 0xffffffff); + + m_ChunkType = chunkType; + + m_Data = AllocAlignedBuffer(m_Length); + + memcpy(m_Data, ser.GetWriter()->GetData(), (size_t)m_Length); + + ser.GetWriter()->Rewind(); + } + + byte *GetData() const { return m_Data; } + Chunk *Duplicate() + { + Chunk *ret = new Chunk(); + ret->m_Length = m_Length; + ret->m_ChunkType = m_ChunkType; + + ret->m_Data = AllocAlignedBuffer(m_Length); + + memcpy(ret->m_Data, m_Data, (size_t)m_Length); + + return ret; + } + + void Write(Serialiser &ser) + { + ser.GetWriter()->Write((const void *)m_Data, (size_t)m_Length); + } private: - Chunk() {} - // no copy semantics - Chunk(const Chunk &); - Chunk &operator=(const Chunk &); + Chunk() = default; + Chunk(const Chunk &) = delete; + Chunk &operator=(const Chunk &) = delete; - friend class ScopedContext; - - bool m_AlignedData; - bool m_Temporary; + friend class ScopedChunk; uint32_t m_ChunkType; uint32_t m_Length; byte *m_Data; - string m_DebugStr; -#if ENABLED(RDOC_DEVEL) +#if !defined(RELEASE) static int64_t m_LiveChunks, m_MaxChunks, m_TotalMem; #endif }; -// this class has a few functions. It can be used to serialise chunks - on writing it enforces -// that we only ever write a single chunk, then pull out the data into a Chunk class and erase -// the contents of the serialiser ready to serialise the next (see the RDCASSERT at the start -// of PushContext). -// -// We use this functionality for sending and receiving data across the network as well as saving -// out to the capture logfile format. -// -// It's also used on reading where it will contain the stream of chunks that were written out -// to the logfile on capture. -// -// When reading, the Serialiser allocates a window of memory and scans through the file by reading -// data into that window and moving along through the file. The window will expand to accomodate -// whichever is the biggest single element within a chunk that's read (so that you can always -// guarantee -// while reading that the element you're interested in is always in memory). -class Serialiser +#ifndef SERIALISER_IMPL +class ScopedChunk { public: - enum Mode + template + ScopedChunk(WriteSerialiser &s, ChunkType i, uint32_t byteLength = 0) + : m_Idx(uint32_t(i)), m_Ser(s), m_Ended(false) { - NONE = 0, - READING, - WRITING, - }; - - enum SerialiserError - { - eSerError_None = 0, - eSerError_FileIO, - eSerError_Corrupt, - eSerError_UnsupportedVersion, - }; - - enum SectionFlags - { - eSectionFlag_None = 0x0, - eSectionFlag_ASCIIStored = 0x1, - eSectionFlag_LZ4Compressed = 0x2, - }; - - enum SectionType - { - eSectionType_Unknown = 0, - eSectionType_FrameCapture, // renderdoc/internal/framecapture - eSectionType_ResolveDatabase, // renderdoc/internal/resolvedb - eSectionType_MachineID, // renderdoc/internal/machineid - eSectionType_FrameBookmarks, // renderdoc/ui/bookmarks - eSectionType_Notes, // renderdoc/ui/notes - eSectionType_Num, - }; - - // version number of overall file format or chunk organisation. If the contents/meaning/order of - // chunks have changed this does not need to be bumped, there are version numbers within each - // API that interprets the stream that can be bumped. - static const uint64_t SERIALISE_VERSION = 0x00000032; - static const uint32_t MAGIC_HEADER; - - ////////////////////////////////////////// - // Init and error handling - - Serialiser(size_t length, const byte *memoryBuf, bool fileheader); - Serialiser(const char *path, Mode mode, bool debugMode, uint64_t sizeHint = 128 * 1024); - ~Serialiser(); - - bool HasError() { return m_HasError; } - SerialiserError ErrorCode() { return m_ErrorCode; } - ////////////////////////////////////////// - // Utility functions - - void *GetUserData() { return m_pUserData; } - void SetUserData(void *userData) { m_pUserData = userData; } - bool AtEnd() { return GetOffset() >= m_BufferSize; } - bool HasAlignedData() { return m_AlignedData; } - bool IsReading() const { return m_Mode == READING; } - bool IsWriting() const { return !IsReading(); } - uint64_t GetOffset() const - { - if(m_HasError) - { - RDCERR("Getting offset with error state serialiser"); - return 0; - } - - RDCASSERT(m_BufferHead && m_Buffer && m_BufferHead >= m_Buffer); - return m_BufferHead - m_Buffer + m_ReadOffset; + m_Ser.WriteChunk(m_Idx, byteLength); } - - uint64_t GetSize() - { - if(m_Mode == READING) - return m_BufferSize; - - return m_BufferHead - m_Buffer; - } - - uint64_t GetFileSize() - { - if(m_Mode == READING) - return m_FileSize; - - return 0; - } - - byte *GetRawPtr(size_t offs) const { return m_Buffer + offs; } - // Set up the base pointer and size. Serialiser will allocate enough for - // the rest of the file and keep it all in memory (useful to keep everything - // in actual frame data resident in memory). - void SetPersistentBlock(uint64_t offs); - - void SetOffset(uint64_t offs); - - void Rewind() - { - m_DebugText = ""; - m_Indent = 0; - m_AlignedData = false; - SetOffset(0); - } - - // assumes buffer head is sitting before a chunk (ie. pushcontext will be valid) - void SkipToChunk(uint32_t chunkIdx, uint32_t *idx = NULL) - { - do - { - size_t offs = m_BufferHead - m_Buffer + (size_t)m_ReadOffset; - - uint32_t c = PushContext(NULL, NULL, 1, false); - - // found - if(c == chunkIdx) - { - m_Indent--; - m_BufferHead = (m_Buffer + offs) - (size_t)m_ReadOffset; - return; - } - else - { - SkipCurrentChunk(); - PopContext(1); - } - - if(idx) - (*idx)++; - - } while(!AtEnd()); - } - - // assumes buffer head is sitting in a chunk (ie. immediately after a pushcontext) - void SkipCurrentChunk() { ReadBytes(m_LastChunkLen); } - void InitCallstackResolver(); - bool HasCallstacks() { return m_KnownSections[eSectionType_ResolveDatabase] != NULL; } - // get callstack resolver, created with the DB in the file - Callstack::StackResolver *GetCallstackResolver() { return m_pResolver; } - void SetCallstack(uint64_t *levels, size_t numLevels); - - uint64_t GetSavedMachineIdent() - { - Section *id = m_KnownSections[eSectionType_MachineID]; - - // section might not be present on older captures (or if it was stripped out - // by someone over-eager to remove information) - if(id != NULL && id->data.size() >= sizeof(uint64_t)) - { - uint64_t ident = 0; - memcpy(&ident, &id->data[0], sizeof(ident)); - return ident; - } - - return 0; - } - - // get the callstack associated with the last scope - Callstack::Stackwalk *GetLastCallstack() - { - return (m_pCallstack && m_pCallstack->NumLevels() > 0) ? m_pCallstack : NULL; - } - - ////////////////////////////////////////// - // Public serialisation interface - - int GetContextLevel() { return m_Indent; } - uint32_t PushContext(const char *name, const char *typeName, uint32_t chunkIdx, bool smallChunk); - void PopContext(uint32_t chunkIdx); - - // Write a chunk to disk - void Insert(Chunk *el); - - // serialise a fixed-size array. - template - void SerialisePODArray(const char *name, T *el) - { - uint32_t n = (uint32_t)Num; - SerialisePODArray(name, el, n); - } - - // serialise a normal array. Typically this should be a small array, - // for large byte buffers use SerialiseBuffer which is optimised for that - // - // If serialising in, el must either be NULL in which case allocated - // memory will be returned, or it must be already large enough. - template - void SerialisePODArray(const char *name, T *&el, uint32_t &numElems) - { - if(m_Mode == WRITING) - { - WriteFrom(numElems); - WriteBytes((byte *)el, sizeof(T) * numElems); - } - else if(m_Mode == READING) - { - ReadInto(numElems); - - if(numElems > 0) - { - if(el == NULL) - el = new T[numElems]; - - size_t length = numElems * sizeof(T); - - memcpy(el, ReadBytes(length), length); - } - } - - if(name != NULL && m_DebugTextWriting) - { - if(numElems == 0) - DebugPrint("%s[]\n", name); - - for(size_t i = 0; i < numElems; i++) - DebugPrint("%s[%d] = %s\n", name, i, ToStr(el[i]).c_str()); - } - } - - // overload for 64-bit counts - template - void SerialisePODArray(const char *name, T *&el, uint64_t &Num) - { - uint32_t n = (uint32_t)Num; - SerialisePODArray(name, el, n); - Num = n; - } - - // serialise a normal array. Typically this should be a small array, - // for large byte buffers use SerialiseBuffer which is optimised for that - // - // If serialising in, el will either be set to NULL or allocated, the - // existing value will be overwritten. - template - void SerialiseComplexArray(const char *name, T *&el) - { - uint32_t n = (uint32_t)Num; - SerialiseComplexArray(name, el, n); - } - - template - void SerialiseComplexArray(const char *name, T *&el, uint32_t &Num) - { - if(m_Mode == WRITING) - { - WriteFrom(Num); - for(uint32_t i = 0; i < Num; i++) - Serialise(m_DebugTextWriting ? StringFormat::Fmt("%s[%i]", name, i).c_str() : "", el[i]); - } - else if(m_Mode == READING) - { - ReadInto(Num); - - if(Num > 0) - { - el = new T[Num]; - - for(uint32_t i = 0; i < Num; i++) - Serialise(m_DebugTextWriting ? StringFormat::Fmt("%s[%i]", name, i).c_str() : "", el[i]); - } - else - { - el = NULL; - } - } - - if(name != NULL && m_DebugTextWriting && Num == 0) - DebugPrint("%s[]\n", name); - } - - // overload for 64-bit counts - template - void SerialiseComplexArray(const char *name, T *&el, uint64_t &Num) - { - uint32_t n = (uint32_t)Num; - SerialiseComplexArray(name, el, n); - Num = n; - } - - // serialise a single element - template - void Serialise(const char *name, T &el) - { - if(m_Mode == WRITING) - { - WriteFrom(el); - } - else if(m_Mode == READING) - { - ReadInto(el); - } - - if(name != NULL && m_DebugTextWriting) - DebugPrint("%s: %s\n", name, ToStr(el).c_str()); - } - - // function to deallocate members - template - void Deserialise(const T *const el) const - { - } - - template - void Serialise(const char *name, std::vector &el) - { - uint64_t sz = el.size(); - Serialise(name, sz); - if(m_Mode == WRITING) - { - for(size_t i = 0; i < sz; i++) - Serialise("[]", el[i]); - } - else - { - el.clear(); - el.reserve((size_t)sz); - for(size_t i = 0; i < sz; i++) - { - X x = X(); - Serialise("", x); - el.push_back(x); - } - } - } - - template - void Serialise(const char *name, rdcarray &el) - { - int32_t sz = el.count(); - Serialise(name, sz); - if(m_Mode == WRITING) - { - for(int32_t i = 0; i < sz; i++) - Serialise("[]", el[i]); - } - else - { - el.resize(sz); - for(int32_t i = 0; i < sz; i++) - Serialise("", el[i]); - } - } - - void Serialise(const char *name, bytebuf &el) - { - int32_t bufLen = el.count(); - - if(m_Mode >= WRITING) - { - WriteFrom(bufLen); - - // ensure byte alignment - uint64_t offs = GetOffset(); - uint64_t alignedoffs = AlignUp(offs, BufferAlignment); - - if(offs != alignedoffs) - { - static const byte padding[64] = {0}; - WriteBytes(&padding[0], (size_t)(alignedoffs - offs)); - } - - RDCASSERT((GetOffset() % BufferAlignment) == 0); - - WriteBytes(el.data(), bufLen); - - m_AlignedData = true; - } - else - { - ReadInto(bufLen); - - // ensure byte alignment - uint64_t offs = GetOffset(); - - // serialise version 0x00000031 had only 16-byte alignment - uint64_t alignedoffs = AlignUp(offs, m_SerVer == 0x00000031 ? 16 : BufferAlignment); - - if(offs != alignedoffs) - { - ReadBytes((size_t)(alignedoffs - offs)); - } - - el.resize((size_t)bufLen); - memcpy(el.data(), ReadBytes(bufLen), bufLen); - } - - if(m_DebugTextWriting && name && name[0]) - { - const char *ellipsis = "..."; - - uint32_t lbuf[4]; - - memcpy(lbuf, el.data(), RDCMIN((size_t)bufLen, 4 * sizeof(uint32_t))); - - if(bufLen <= 16) - { - ellipsis = " "; - } - - DebugPrint("%s: RawBuffer % 5d:< 0x%08x 0x%08x 0x%08x 0x%08x %s>\n", name, bufLen, lbuf[0], - lbuf[1], lbuf[2], lbuf[3], ellipsis); - } - } - - void Serialise(const char *name, rdcstr &el) - { - int32_t sz = el.count(); - Serialise(name, sz); - if(m_Mode == WRITING) - { - for(int32_t i = 0; i < sz; i++) - Serialise("[]", el[i]); - } - else - { - el.resize((size_t)sz); - for(int32_t i = 0; i < sz; i++) - Serialise("", el[i]); - } - } - - template - void Serialise(const char *name, std::pair &el) - { - Serialise(name, el.first); - Serialise(name, el.second); - } - - template - void Serialise(const char *name, rdcpair &el) - { - Serialise(name, el.first); - Serialise(name, el.second); - } - - template - void Serialise(const char *name, std::list &el) - { - uint64_t sz = el.size(); - Serialise(name, sz); - if(m_Mode == WRITING) - { - for(auto it = el.begin(); it != el.end(); ++it) - Serialise("[]", *it); - } - else - { - el.clear(); - for(uint64_t i = 0; i < sz; i++) - { - X x = X(); - Serialise("", x); - el.push_back(x); - } - } - } - - // not sure if I still neeed these specialisations anymore. - void SerialiseString(const char *name, string &el); - - // serialise a buffer. - // - // If serialising in, buf must either be NULL in which case allocated - // memory will be returned, or it must be already large enough. - void SerialiseBuffer(const char *name, byte *&buf, size_t &len); - void AlignNextBuffer(const size_t alignment); - - // NOT recommended interface. Useful for specific situations if e.g. you have - // a buffer of data that is not arbitrary in size and can be determined by a 'type' or - // similar elsewhere in the stream, so you want to skip the type-safety of the above - // and write directly into the stream. Must be matched by a RawReadBytes. - void RawWriteBytes(const void *data, size_t bytes) { WriteBytes((const byte *)data, bytes); } - const void *RawReadBytes(size_t bytes) { return ReadBytes(bytes); } - // prints to the debug output log - void DebugPrint(const char *fmt, ...); - - void FlushToDisk(); - - // set a function used when serialising a text representation - // of the chunks - void SetChunkNameLookup(ChunkLookup lookup) { m_ChunkLookup = lookup; } - void SetDebugText(bool enabled) { m_DebugTextWriting = enabled; } - bool GetDebugText() { return m_DebugTextWriting; } - string GetDebugStr() { return m_DebugText; } -private: - ////////////////////////////////////////// - // Raw memory buffer read/write - - void WriteBytes(const byte *buf, size_t nBytes); - void *ReadBytes(size_t nBytes); - - void ReadFromFile(uint64_t bufferOffs, size_t length); - - template - void WriteFrom(const T &f) - { - WriteBytes((byte *)&f, sizeof(T)); - } - - template - void ReadInto(T &f) - { - if(m_HasError) - { - RDCERR("Reading into with error state serialiser"); - return; - } - - char *data = (char *)ReadBytes(sizeof(T)); -#if defined(_M_ARM) || defined(__arm__) - // Fetches on ARM have to be aligned according to the type size. - memcpy(&f, data, sizeof(T)); -#else - f = *((T *)data); -#endif - } - - // no copies - Serialiser(const Serialiser &other); - - void CreateResolver(); - - // clean out for before constructor and after destructor (and other times probably) - void Reset(); - - string GetIndent() - { - if(m_Mode == READING) - return string(m_Indent > 0 ? 4 : 0, ' '); - - return string((size_t)m_Indent * 4, ' '); - } - - ////////////////////////////////////////// - - static const uint64_t BufferAlignment; - - ////////////////////////////////////////// - - uint64_t m_SerVer; - - Mode m_Mode; - - SerialiserError m_ErrorCode; - bool m_HasError; - bool m_DebugEnabled; - - void *m_pUserData; - - int m_Indent; - - Callstack::Stackwalk *m_pCallstack; - Callstack::StackResolver *m_pResolver; - Threading::ThreadHandle m_ResolverThread; - volatile bool m_ResolverThreadKillSignal; - - string m_Filename; - - // raw binary buffer - uint64_t m_BufferSize; - byte *m_Buffer; - byte *m_BufferHead; - size_t m_LastChunkLen; - bool m_AlignedData; - vector m_ChunkFixups; - - // reading from file: - - struct Section - { - Section() - : type(eSectionType_Unknown), flags(eSectionFlag_None), fileoffset(0), compressedReader(NULL) - { - } - string name; - SectionType type; - SectionFlags flags; - - uint64_t fileoffset; - uint64_t size; - vector data; // some sections can be loaded entirely into memory - CompressedFileIO *compressedReader; - }; - - // this lists all sections in file order - vector
m_Sections; - - // this lists known sections, some may be NULL - Section *m_KnownSections[eSectionType_Num]; - - // where does our in-memory window point to in the data stream. ie. m_pBuffer[0] is - // m_ReadOffset into the frame capture section - uint64_t m_ReadOffset; - - uint64_t m_FileSize; - - // how big is the current in-memory window - size_t m_CurrentBufferSize; - - // the file pointer to read from - FILE *m_ReadFileHandle; - - // writing to file - vector m_Chunks; - - // a database of strings read from the file, useful when serialised structures - // expect a char* to return and point to static memory - set m_StringDB; - - // debug buffer - bool m_DebugTextWriting; - string m_DebugText; - ChunkLookup m_ChunkLookup; -}; - -template <> -void Serialiser::Serialise(const char *name, string &el); - -// floats need aligned reads -template <> -void Serialiser::ReadInto(float &f); - -class ScopedContext -{ -public: - ScopedContext(Serialiser *s, const char *n, const char *t, uint32_t i, bool smallChunk) - : m_Idx(i), m_Ser(s), m_Ended(false) - { - m_Ser->PushContext(n, t, m_Idx, smallChunk); - } - ScopedContext(Serialiser *s, const char *n, uint32_t i, bool smallChunk) - : m_Idx(i), m_Ser(s), m_Ended(false) - { - m_Ser->PushContext(n, NULL, m_Idx, smallChunk); - } - ~ScopedContext() + ~ScopedChunk() { if(!m_Ended) End(); } - Chunk *Get(bool temporary = false) + Chunk *Get() { End(); - return new Chunk(m_Ser, m_Idx, temporary); + return new Chunk(m_Ser, m_Idx); } private: + WriteSerialiser &m_Ser; uint32_t m_Idx; - Serialiser *m_Ser; - bool m_Ended; void End() { RDCASSERT(!m_Ended); - m_Ser->PopContext(m_Idx); + m_Ser.EndChunk(); m_Ended = true; } }; +#endif -template +template struct ScopedDeserialise { - ScopedDeserialise(const Serialiser *const ser, const T *const t) : m_ser(ser), m_t(t) {} - ~ScopedDeserialise() { m_ser->Deserialise(m_t); } - const Serialiser *const m_ser; - const T *const m_t; + ScopedDeserialise(const SerialiserType &ser, const T &el) : m_Ser(ser), m_El(el) {} + ~ScopedDeserialise() + { + if(m_Ser.IsReading()) + Deserialise(m_El); + } + const SerialiserType &m_Ser; + const T &m_El; }; -// can be overridden to locally cache the serialiser pointer (e.g. for TLS lookup) -#define GET_SERIALISER GetSerialiser() +template +struct ScopedDeserialiseNullable +{ + typedef typename std::remove_pointer::type T; -#define SCOPED_SERIALISE_CONTEXT(n) ScopedContext scope(GET_SERIALISER, GetChunkName(n), n, false); -#define SCOPED_SERIALISE_CONTEXT(n) ScopedContext scope(GET_SERIALISER, GetChunkName(n), n, false); -#define SCOPED_SERIALISE_SMALL_CONTEXT(n) \ - ScopedContext scope(GET_SERIALISER, GetChunkName(n), n, true); + ScopedDeserialiseNullable(const SerialiserType &ser, T **el) : m_Ser(ser), m_El(el) {} + ~ScopedDeserialiseNullable() + { + if(m_Ser.IsReading() && *m_El != NULL) + { + Deserialise(**m_El); + delete *m_El; + } + } + const SerialiserType &m_Ser; + T **m_El; +}; -#define SERIALISE_ELEMENT(type, name, inValue) \ - type name; \ - ScopedDeserialise CONCAT(deserialise_, name)(GET_SERIALISER, &name); \ - if(m_State >= WRITING) \ - name = (inValue); \ - GET_SERIALISER->Serialise(#name, name); -#define SERIALISE_ELEMENT_OPT(type, name, inValue, Condition) \ - type name = type(); \ - if(Condition) \ - { \ - if(m_State >= WRITING) \ - name = (inValue); \ - GET_SERIALISER->Serialise(#name, name); \ +template +struct ScopedDeserialiseArray +{ + typedef typename std::remove_pointer::type T; + + ScopedDeserialiseArray(const SerialiserType &ser, T **el) : m_Ser(ser), m_El(el) {} + ~ScopedDeserialiseArray() + { + if(m_Ser.IsReading()) + delete[] * m_El; } -#define SERIALISE_ELEMENT_ARR(type, name, inValues, count) \ - type *name = new type[count]; \ - for(size_t serialiseIdx = 0; serialiseIdx < count; serialiseIdx++) \ - { \ - if(m_State >= WRITING) \ - name[serialiseIdx] = (inValues)[serialiseIdx]; \ - GET_SERIALISER->Serialise(#name, name[serialiseIdx]); \ + const SerialiserType &m_Ser; + T **m_El; +}; + +template +struct ScopedDeserialiseArray +{ + ScopedDeserialiseArray(const SerialiserType &ser, void **el) : m_Ser(ser), m_El(el) {} + ~ScopedDeserialiseArray() + { + if(m_Ser.IsReading()) + FreeAlignedBuffer((byte *)*m_El); } -#define SERIALISE_ELEMENT_ARR_OPT(type, name, inValues, count, Condition) \ - type *name = NULL; \ - if(Condition) \ - { \ - name = new type[count]; \ - for(size_t serialiseIdx = 0; serialiseIdx < count; serialiseIdx++) \ - { \ - if(m_State >= WRITING) \ - name[serialiseIdx] = (inValues)[serialiseIdx]; \ - GET_SERIALISER->Serialise(#name, name[serialiseIdx]); \ - } \ + const SerialiserType &m_Ser; + void **m_El; +}; + +template +struct ScopedDeserialiseArray +{ + ScopedDeserialiseArray(const SerialiserType &ser, const void **el) : m_Ser(ser), m_El(el) {} + ~ScopedDeserialiseArray() + { + if(m_Ser.IsReading()) + FreeAlignedBuffer((byte *)*m_El); } -#define SERIALISE_ELEMENT_PTR(type, name, inValue) \ - type name; \ - if(inValue && m_State >= WRITING) \ - name = *(inValue); \ - GET_SERIALISER->Serialise(#name, name); -#define SERIALISE_ELEMENT_PTR_OPT(type, name, inValue, Condition) \ - type name; \ - if(Condition) \ - { \ - if(inValue && m_State >= WRITING) \ - name = *(inValue); \ - GET_SERIALISER->Serialise(#name, name); \ - } -#define SERIALISE_ELEMENT_BUF(type, name, inBuf, Len) \ - type name = (type)NULL; \ - if(m_State >= WRITING) \ - name = (type)(inBuf); \ - size_t CONCAT(buflen, __LINE__) = Len; \ - GET_SERIALISER->SerialiseBuffer(#name, name, CONCAT(buflen, __LINE__)); -#define SERIALISE_ELEMENT_BUF_OPT(type, name, inBuf, Len, Condition) \ - type name = (type)NULL; \ - if(Condition) \ - { \ - if(m_State >= WRITING) \ - name = (type)(inBuf); \ - size_t CONCAT(buflen, __LINE__) = Len; \ - GET_SERIALISER->SerialiseBuffer(#name, name, CONCAT(buflen, __LINE__)); \ + const SerialiserType &m_Ser; + const void **m_El; +}; + +template +struct ScopedDeserialiseArray +{ + ScopedDeserialiseArray(const SerialiserType &ser, byte **el) : m_Ser(ser), m_El(el) {} + ~ScopedDeserialiseArray() + { + if(m_Ser.IsReading()) + FreeAlignedBuffer(*m_El); } + const SerialiserType &m_Ser; + byte **m_El; +}; + +// can be overridden to change the name in the helper macros locally +#define GET_SERIALISER (ser) + +#define SCOPED_SERIALISE_CHUNK(...) ScopedChunk scope(GET_SERIALISER, __VA_ARGS__); + +// these helper macros are intended for use when serialising objects in chunks +#define SERIALISE_ELEMENT(obj) \ + ScopedDeserialise CONCAT(deserialise_, __LINE__)( \ + GET_SERIALISER, obj); \ + GET_SERIALISER.Serialise(#obj, obj) + +#define SERIALISE_ELEMENT_TYPED(type, obj) \ + if(ser.IsReading()) \ + obj = decltype(obj)(); \ + union \ + { \ + type *t; \ + decltype(obj) *o; \ + } CONCAT(union, __LINE__); \ + CONCAT(union, __LINE__).o = &obj; \ + GET_SERIALISER.template Serialise(#obj, *CONCAT(union, __LINE__).t) + +// helper macro for when an array has a constant size. Uses a dummy variable to serialise +#define FIXED_COUNT(count) \ + (CONCAT(dummy_array_count, __LINE__) = (count), CONCAT(dummy_array_count, __LINE__)) + +#define SERIALISE_ELEMENT_ARRAY(obj, count) \ + uint64_t CONCAT(dummy_array_count, __LINE__) = 0; \ + (void)CONCAT(dummy_array_count, __LINE__); \ + ScopedDeserialiseArray CONCAT(deserialise_, __LINE__)( \ + GET_SERIALISER, &obj); \ + GET_SERIALISER.Serialise(#obj, obj, count, SerialiserFlags::AllocateMemory) + +#define SERIALISE_ELEMENT_OPT(obj) \ + ScopedDeserialiseNullable CONCAT( \ + deserialise_, __LINE__)(GET_SERIALISER, &obj); \ + GET_SERIALISER.SerialiseNullable(#obj, obj) + +#define SERIALISE_ELEMENT_LOCAL(obj, inValue) \ + typename std::remove_cv::type>::type obj; \ + ScopedDeserialise CONCAT(deserialise_, __LINE__)( \ + GET_SERIALISER, obj); \ + if(GET_SERIALISER.IsWriting()) \ + obj = (inValue); \ + GET_SERIALISER.Serialise(#obj, obj) + +// these macros are for use when implementing a DoSerialise function +#define SERIALISE_MEMBER(obj) ser.Serialise(#obj, el.obj) + +#define SERIALISE_MEMBER_TYPED(type, obj) \ + if(ser.IsReading()) \ + el.obj = decltype(el.obj)(); \ + union \ + { \ + type *t; \ + decltype(el.obj) *o; \ + } CONCAT(union, __LINE__); \ + CONCAT(union, __LINE__).o = &el.obj; \ + ser.template Serialise(#obj, *CONCAT(union, __LINE__).t) + +#define SERIALISE_MEMBER_ARRAY(arrayObj, countObj) \ + ser.Serialise(#arrayObj, el.arrayObj, el.countObj, SerialiserFlags::AllocateMemory) + +// a member that is a pointer and could be NULL, so needs a hidden 'present' +// flag serialised out +#define SERIALISE_MEMBER_OPT(obj) ser.SerialiseNullable(#obj, el.obj) diff --git a/renderdoc/serialise/serialiser_tests.cpp b/renderdoc/serialise/serialiser_tests.cpp new file mode 100644 index 000000000..576689d43 --- /dev/null +++ b/renderdoc/serialise/serialiser_tests.cpp @@ -0,0 +1,1320 @@ +/****************************************************************************** + * The MIT License (MIT) + * + * Copyright (c) 2017 Baldur Karlsson + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + ******************************************************************************/ + +#include "serialiser.h" + +#if ENABLED(ENABLE_UNIT_TESTS) + +#include "3rdparty/catch/catch.hpp" + +void WriteAllBasicTypes(WriteSerialiser &ser) +{ + int64_t a = -1; + uint64_t b = 2; + int32_t c = -3; + uint32_t d = 4; + int16_t e = -5; + uint16_t f = 6; + int8_t g = -7; + uint8_t h = 8; + + bool i = true; + + char j = 'j'; + + double k = 11.11011011; + float l = 12.12012012f; + + std::string m = "mmmm"; + char n[5] = "nnnn"; + const char *s = "ssss"; + + int t[4] = {20, 20, 20, 20}; + + SERIALISE_ELEMENT(a); + SERIALISE_ELEMENT(b); + SERIALISE_ELEMENT(c); + SERIALISE_ELEMENT(d); + SERIALISE_ELEMENT(e); + SERIALISE_ELEMENT(f); + SERIALISE_ELEMENT(g); + SERIALISE_ELEMENT(h); + SERIALISE_ELEMENT(i); + SERIALISE_ELEMENT(j); + SERIALISE_ELEMENT(k); + SERIALISE_ELEMENT(l); + SERIALISE_ELEMENT(m); + SERIALISE_ELEMENT(n); + SERIALISE_ELEMENT(s); + SERIALISE_ELEMENT(t); +} + +TEST_CASE("Read/write basic types", "[serialiser][structured]") +{ + StreamWriter *buf = new StreamWriter(StreamWriter::DefaultScratchSize); + + // write basic types, verify that we didn't write too much (rough factor of total data size + + // overhead - it's OK to update this value if serialisation changed as long as it's incremental). + { + WriteSerialiser ser(buf, Ownership::Nothing); + + { + SCOPED_SERIALISE_CHUNK(5); + + WriteAllBasicTypes(ser); + } + + CHECK(buf->GetOffset() <= 128); + + REQUIRE_FALSE(ser.IsErrored()); + } + + { + ReadSerialiser ser(new StreamReader(buf->GetData(), buf->GetOffset()), Ownership::Stream); + + uint32_t chunkID = ser.ReadChunk(); + + CHECK(chunkID == 5); + + int64_t a; + uint64_t b; + int32_t c; + uint32_t d; + int16_t e; + uint16_t f; + int8_t g; + uint8_t h; + + bool i; + + char j; + + double k; + float l; + + std::string m; + char n[5]; + const char *s; + + int t[4] = {0, 0, 0, 0}; + + SERIALISE_ELEMENT(a); + SERIALISE_ELEMENT(b); + SERIALISE_ELEMENT(c); + SERIALISE_ELEMENT(d); + SERIALISE_ELEMENT(e); + SERIALISE_ELEMENT(f); + SERIALISE_ELEMENT(g); + SERIALISE_ELEMENT(h); + SERIALISE_ELEMENT(i); + SERIALISE_ELEMENT(j); + SERIALISE_ELEMENT(k); + SERIALISE_ELEMENT(l); + SERIALISE_ELEMENT(m); + SERIALISE_ELEMENT(n); + SERIALISE_ELEMENT(s); + SERIALISE_ELEMENT(t); + + ser.EndChunk(); + + REQUIRE_FALSE(ser.IsErrored()); + + CHECK(ser.GetReader()->AtEnd()); + + CHECK(a == -1); + CHECK(b == 2); + CHECK(c == -3); + CHECK(d == 4); + CHECK(e == -5); + CHECK(f == 6); + CHECK(g == -7); + CHECK(h == 8); + + CHECK(i == true); + + CHECK(j == 'j'); + + CHECK(k == 11.11011011); + CHECK(l == 12.12012012f); + + CHECK(m == "mmmm"); + CHECK(std::string(n) == "nnnn"); + CHECK(std::string(s) == "ssss"); + + CHECK(t[0] == 20); + CHECK(t[1] == 20); + CHECK(t[2] == 20); + CHECK(t[3] == 20); + } + + delete buf; +}; + +TEST_CASE("Read/write via structured of basic types", "[serialiser]") +{ + StreamWriter *buf = new StreamWriter(StreamWriter::DefaultScratchSize); + + // write basic types, verify that we didn't write too much (rough factor of total data size + + // overhead - it's OK to update this value if serialisation changed as long as it's incremental). + { + WriteSerialiser ser(buf, Ownership::Nothing); + ser.WriteChunk(5); + WriteAllBasicTypes(ser); + ser.EndChunk(); + } + + { + ReadSerialiser ser(new StreamReader(buf->GetData(), buf->GetOffset()), Ownership::Stream); + + ChunkLookup testChunkLoop = [](uint32_t) -> std::string { return "TestChunk"; }; + + ser.ConfigureStructuredExport(testChunkLoop, true); + + int64_t a; + uint64_t b; + int32_t c; + uint32_t d; + int16_t e; + uint16_t f; + int8_t g; + uint8_t h; + + bool i; + + char j; + + double k; + float l; + + std::string m; + char n[5]; + const char *s; + + int t[4]; + + ser.ReadChunk(); + + SERIALISE_ELEMENT(a); + SERIALISE_ELEMENT(b); + SERIALISE_ELEMENT(c); + SERIALISE_ELEMENT(d); + SERIALISE_ELEMENT(e); + SERIALISE_ELEMENT(f); + SERIALISE_ELEMENT(g); + SERIALISE_ELEMENT(h); + SERIALISE_ELEMENT(i); + SERIALISE_ELEMENT(j); + SERIALISE_ELEMENT(k); + SERIALISE_ELEMENT(l); + SERIALISE_ELEMENT(m); + SERIALISE_ELEMENT(n); + SERIALISE_ELEMENT(s); + SERIALISE_ELEMENT(t); + + ser.EndChunk(); + + REQUIRE_FALSE(ser.IsErrored()); + + CHECK(ser.GetReader()->AtEnd()); + + const SDFile &structFile = ser.GetStructuredFile(); + + CHECK(structFile.chunks.size() == 1); + CHECK(structFile.buffers.size() == 0); + + REQUIRE(structFile.chunks[0]); + + const SDChunk &chunk = *structFile.chunks[0]; + + CHECK(chunk.name == testChunkLoop(0)); + CHECK(chunk.metadata.chunkID == 5); + CHECK(chunk.metadata.length == chunk.type.byteSize); + CHECK(chunk.metadata.length < ser.GetReader()->GetSize()); + CHECK(chunk.type.basetype == SDBasic::Chunk); + CHECK(chunk.type.name == "Chunk"); + CHECK(chunk.data.children.size() == 16); + + for(SDObject *o : chunk.data.children) + REQUIRE(o); + + int childIdx = 0; + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "a"); + CHECK(o.type.name == "int64_t"); + CHECK(o.type.basetype == SDBasic::SignedInteger); + CHECK(o.type.byteSize == 8); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + + CHECK(o.data.basic.i == -1); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "b"); + CHECK(o.type.name == "uint64_t"); + CHECK(o.type.basetype == SDBasic::UnsignedInteger); + CHECK(o.type.byteSize == 8); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + + CHECK(o.data.basic.u == 2); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "c"); + CHECK(o.type.name == "int32_t"); + CHECK(o.type.basetype == SDBasic::SignedInteger); + CHECK(o.type.byteSize == 4); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + + CHECK(o.data.basic.i == -3); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "d"); + CHECK(o.type.name == "uint32_t"); + CHECK(o.type.basetype == SDBasic::UnsignedInteger); + CHECK(o.type.byteSize == 4); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + + CHECK(o.data.basic.u == 4); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "e"); + CHECK(o.type.name == "int16_t"); + CHECK(o.type.basetype == SDBasic::SignedInteger); + CHECK(o.type.byteSize == 2); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + + CHECK(o.data.basic.i == -5); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "f"); + CHECK(o.type.name == "uint16_t"); + CHECK(o.type.basetype == SDBasic::UnsignedInteger); + CHECK(o.type.byteSize == 2); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + + CHECK(o.data.basic.u == 6); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "g"); + CHECK(o.type.name == "int8_t"); + CHECK(o.type.basetype == SDBasic::SignedInteger); + CHECK(o.type.byteSize == 1); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + + CHECK(o.data.basic.i == -7); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "h"); + CHECK(o.type.name == "uint8_t"); + CHECK(o.type.basetype == SDBasic::UnsignedInteger); + CHECK(o.type.byteSize == 1); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + + CHECK(o.data.basic.u == 8); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "i"); + CHECK(o.type.name == "bool"); + CHECK(o.type.basetype == SDBasic::Boolean); + CHECK(o.type.byteSize == 1); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + + CHECK(o.data.basic.b == true); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "j"); + CHECK(o.type.name == "char"); + CHECK(o.type.basetype == SDBasic::Character); + CHECK(o.type.byteSize == 1); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + + CHECK(o.data.basic.c == 'j'); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "k"); + CHECK(o.type.name == "double"); + CHECK(o.type.basetype == SDBasic::Float); + CHECK(o.type.byteSize == 8); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + + CHECK(o.data.basic.d == 11.11011011); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "l"); + CHECK(o.type.name == "float"); + CHECK(o.type.basetype == SDBasic::Float); + CHECK(o.type.byteSize == 4); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + + CHECK(o.data.basic.d == Approx(12.12012012)); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "m"); + CHECK(o.type.name == "string"); + CHECK(o.type.basetype == SDBasic::String); + CHECK(o.type.byteSize == 4); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + + CHECK(o.data.str == "mmmm"); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "n"); + CHECK(o.type.name == "string"); + CHECK(o.type.basetype == SDBasic::String); + CHECK(o.type.byteSize == 4); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + + CHECK(o.data.str == "nnnn"); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "s"); + CHECK(o.type.name == "const char *"); + CHECK(o.type.basetype == SDBasic::String); + CHECK(o.type.byteSize == 4); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + + CHECK(o.data.str == "ssss"); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "t"); + CHECK(o.type.name == "int32_t"); + CHECK(o.type.basetype == SDBasic::Array); + CHECK(o.type.byteSize == 4); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + + CHECK(o.data.children.size() == 4); + + CHECK(o.data.children[0]->data.basic.i == 20); + CHECK(o.data.children[1]->data.basic.c == 20); + CHECK(o.data.children[2]->data.basic.c == 20); + CHECK(o.data.children[3]->data.basic.c == 20); + } + + StreamWriter *rewriteBuf = new StreamWriter(StreamWriter::DefaultScratchSize); + + { + WriteSerialiser rewrite(rewriteBuf, Ownership::Nothing); + + rewrite.WriteStructuredFile(structFile); + } + + // must be bitwise identical to the original serialised data. + REQUIRE(rewriteBuf->GetOffset() == buf->GetOffset()); + CHECK_FALSE(memcmp(rewriteBuf->GetData(), buf->GetData(), (size_t)rewriteBuf->GetOffset())); + + delete rewriteBuf; + } + + delete buf; +}; + +TEST_CASE("Read/write chunk metadata", "[serialiser]") +{ + StreamWriter *buf = new StreamWriter(StreamWriter::DefaultScratchSize); + + // write basic types, verify that we didn't write too much (rough factor of total data size + + // overhead - it's OK to update this value if serialisation changed as long as it's incremental). + { + WriteSerialiser ser(buf, Ownership::Nothing); + + ser.SetChunkMetadataRecording(WriteSerialiser::ChunkCallstack | WriteSerialiser::ChunkDuration | + WriteSerialiser::ChunkThreadID | WriteSerialiser::ChunkTimestamp); + + ser.ChunkMetadata().threadID = 12345ULL; + ser.ChunkMetadata().durationMicro = 445566ULL; + ser.ChunkMetadata().timestampMicro = 987654321ULL; + ser.ChunkMetadata().callstack.resize(4); + ser.ChunkMetadata().callstack[0] = 101; + ser.ChunkMetadata().callstack[1] = 102; + ser.ChunkMetadata().callstack[2] = 103; + ser.ChunkMetadata().callstack[3] = 104; + + ser.WriteChunk(1); + + uint32_t dummy = 99; + ser.Serialise("dummy", dummy); + + ser.EndChunk(); + + REQUIRE_FALSE(ser.IsErrored()); + } + + { + ReadSerialiser ser(new StreamReader(buf->GetData(), buf->GetOffset()), Ownership::Stream); + + ser.ReadChunk(); + + CHECK(ser.ChunkMetadata().threadID == 12345ULL); + CHECK(ser.ChunkMetadata().durationMicro == 445566ULL); + CHECK(ser.ChunkMetadata().timestampMicro == 987654321ULL); + REQUIRE(ser.ChunkMetadata().callstack.size() == 4); + CHECK(ser.ChunkMetadata().callstack[0] == 101); + CHECK(ser.ChunkMetadata().callstack[1] == 102); + CHECK(ser.ChunkMetadata().callstack[2] == 103); + CHECK(ser.ChunkMetadata().callstack[3] == 104); + + ser.SkipCurrentChunk(); + + ser.EndChunk(); + + REQUIRE_FALSE(ser.IsErrored()); + + CHECK(ser.GetReader()->AtEnd()); + } + + { + ReadSerialiser ser(new StreamReader(buf->GetData(), buf->GetOffset()), Ownership::Stream); + + ChunkLookup testChunkLoop = [](uint32_t) -> std::string { return "TestChunk"; }; + + ser.ConfigureStructuredExport(testChunkLoop, true); + + ser.ReadChunk(); + + uint32_t dummy; + ser.Serialise("dummy", dummy); + + ser.EndChunk(); + + SDChunkMetaData &md = ser.GetStructuredFile().chunks[0]->metadata; + + CHECK(md.threadID == 12345ULL); + CHECK(md.durationMicro == 445566ULL); + CHECK(md.timestampMicro == 987654321ULL); + REQUIRE(md.callstack.size() == 4); + CHECK(md.callstack[0] == 101); + CHECK(md.callstack[1] == 102); + CHECK(md.callstack[2] == 103); + CHECK(md.callstack[3] == 104); + + REQUIRE_FALSE(ser.IsErrored()); + + CHECK(ser.GetReader()->AtEnd()); + } + + delete buf; +}; + +TEST_CASE("Verify multiple chunks can be merged", "[serialiser][chunks]") +{ + StreamWriter *buf = new StreamWriter(StreamWriter::DefaultScratchSize); + + enum ChunkType + { + FLOAT4 = 5, + TWO_INTS, + BOOL_INT_FLOAT, + STRING_AND_INT, + }; + + // write some chunks individually + std::vector chunks; + { + WriteSerialiser ser(new StreamWriter(StreamWriter::DefaultScratchSize), Ownership::Stream); + + { + SCOPED_SERIALISE_CHUNK(TWO_INTS); + + int first = 123; + int second = 456; + + SERIALISE_ELEMENT(first); + SERIALISE_ELEMENT(second); + + chunks.push_back(scope.Get()); + REQUIRE(chunks.back()); + } + + { + SCOPED_SERIALISE_CHUNK(STRING_AND_INT); + + string s = "string in STRING_AND_INT"; + int i = 4096; + + SERIALISE_ELEMENT(s); + SERIALISE_ELEMENT(i); + + chunks.push_back(scope.Get()); + REQUIRE(chunks.back()); + } + + REQUIRE_FALSE(ser.IsErrored()); + REQUIRE(chunks.size() == 2); + } + + // now write the previous chunks, then some more in-line + { + WriteSerialiser ser(buf, Ownership::Nothing); + + for(Chunk *c : chunks) + c->Write(ser); + + { + SCOPED_SERIALISE_CHUNK(BOOL_INT_FLOAT); + + bool flag = false; + int data = 10000; + float value = 3.141592f; + + SERIALISE_ELEMENT(flag); + SERIALISE_ELEMENT(data); + SERIALISE_ELEMENT(value); + } + + { + SCOPED_SERIALISE_CHUNK(FLOAT4); + + float vec4[4] = {1.1f, 2.2f, 3.3f, 4.4f}; + + SERIALISE_ELEMENT(vec4); + } + + REQUIRE_FALSE(ser.IsErrored()); + CHECK(buf->GetOffset() <= 256); + } + + for(Chunk *c : chunks) + delete c; + + // now read the data "dynamically" and ensure it's all correct + { + ReadSerialiser ser(new StreamReader(buf->GetData(), buf->GetOffset()), Ownership::Stream); + + while(!ser.GetReader()->AtEnd()) + { + uint32_t chunkID = ser.ReadChunk(); + ChunkType chunk = (ChunkType)chunkID; + + switch(chunk) + { + case FLOAT4: + { + float vec4[4]; + + SERIALISE_ELEMENT(vec4); + + CHECK(vec4[0] == 1.1f); + CHECK(vec4[1] == 2.2f); + CHECK(vec4[2] == 3.3f); + CHECK(vec4[3] == 4.4f); + break; + } + case TWO_INTS: + { + int first = 0; + int second = 0; + + SERIALISE_ELEMENT(first); + SERIALISE_ELEMENT(second); + + CHECK(first == 123); + CHECK(second == 456); + break; + } + case BOOL_INT_FLOAT: + { + bool flag = true; + int data = 0; + float value = 0.0f; + + SERIALISE_ELEMENT(flag); + SERIALISE_ELEMENT(data); + SERIALISE_ELEMENT(value); + + CHECK(flag == false); + CHECK(data == 10000); + CHECK(value == 3.141592f); + break; + } + case STRING_AND_INT: + { + string s; + int i = 0; + + SERIALISE_ELEMENT(s); + SERIALISE_ELEMENT(i); + + CHECK(s == "string in STRING_AND_INT"); + CHECK(i == 4096); + break; + } + default: + { + CAPTURE(chunkID); + FAIL("Unexpected chunk type"); + break; + } + } + + ser.EndChunk(); + } + } + + delete buf; +}; + +TEST_CASE("Read/write container types", "[serialiser][structured]") +{ + StreamWriter *buf = new StreamWriter(StreamWriter::DefaultScratchSize); + + // write basic types, verify that we didn't write too much (rough factor of total data size + + // overhead - it's OK to update this value if serialisation changed as long as it's incremental). + { + WriteSerialiser ser(buf, Ownership::Nothing); + + { + SCOPED_SERIALISE_CHUNK(5); + + std::vector v; + std::pair p; + std::list l; + + v.push_back(1); + v.push_back(1); + v.push_back(2); + v.push_back(3); + v.push_back(5); + v.push_back(8); + + p = std::make_pair(3.14159f, "M_PI"); + + l.push_back(2U); + l.push_back(3U); + l.push_back(5U); + l.push_back(7U); + l.push_back(11U); + l.push_back(13U); + + SERIALISE_ELEMENT(v); + SERIALISE_ELEMENT(p); + SERIALISE_ELEMENT(l); + } + + CHECK(buf->GetOffset() <= 128); + + REQUIRE_FALSE(ser.IsErrored()); + } + + { + ReadSerialiser ser(new StreamReader(buf->GetData(), buf->GetOffset()), Ownership::Stream); + + uint32_t chunkID = ser.ReadChunk(); + + CHECK(chunkID == 5); + + std::vector v; + std::pair p; + std::list l; + + SERIALISE_ELEMENT(v); + SERIALISE_ELEMENT(p); + SERIALISE_ELEMENT(l); + + ser.EndChunk(); + + REQUIRE_FALSE(ser.IsErrored()); + + CHECK(ser.GetReader()->AtEnd()); + + REQUIRE(v.size() == 6); + REQUIRE(l.size() == 6); + + CHECK(v[0] == 1); + CHECK(v[1] == 1); + CHECK(v[2] == 2); + CHECK(v[3] == 3); + CHECK(v[4] == 5); + CHECK(v[5] == 8); + + CHECK(p.first == 3.14159f); + CHECK(p.second == "M_PI"); + + auto it = l.begin(); + + CHECK(*it == 2U); + ++it; + CHECK(*it == 3U); + ++it; + CHECK(*it == 5U); + ++it; + CHECK(*it == 7U); + ++it; + CHECK(*it == 11U); + ++it; + CHECK(*it == 13U); + } + + { + ReadSerialiser ser(new StreamReader(buf->GetData(), buf->GetOffset()), Ownership::Stream); + + ser.ConfigureStructuredExport([](uint32_t) -> std::string { return "TestChunk"; }, true); + + ser.ReadChunk(); + { + std::vector v; + std::pair p; + std::list l; + + SERIALISE_ELEMENT(v); + SERIALISE_ELEMENT(p); + SERIALISE_ELEMENT(l); + } + ser.EndChunk(); + + REQUIRE_FALSE(ser.IsErrored()); + + CHECK(ser.GetReader()->AtEnd()); + + const SDFile &structData = ser.GetStructuredFile(); + + CHECK(structData.chunks.size() == 1); + CHECK(structData.buffers.size() == 0); + + REQUIRE(structData.chunks[0]); + + const SDChunk &chunk = *structData.chunks[0]; + + CHECK(chunk.data.children.size() == 3); + + for(SDObject *o : chunk.data.children) + REQUIRE(o); + + int childIdx = 0; + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "v"); + CHECK(o.type.basetype == SDBasic::Array); + CHECK(o.type.byteSize == 6); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + CHECK(o.data.children.size() == 6); + + for(SDObject *child : o.data.children) + { + CHECK(child->type.basetype == SDBasic::SignedInteger); + CHECK(child->type.byteSize == 4); + } + + CHECK(o.data.children[0]->data.basic.i == 1); + CHECK(o.data.children[1]->data.basic.i == 1); + CHECK(o.data.children[2]->data.basic.i == 2); + CHECK(o.data.children[3]->data.basic.i == 3); + CHECK(o.data.children[4]->data.basic.i == 5); + CHECK(o.data.children[5]->data.basic.i == 8); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "p"); + CHECK(o.type.name == "pair"); + CHECK(o.type.basetype == SDBasic::Struct); + CHECK(o.type.byteSize == 2); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + CHECK(o.data.children.size() == 2); + + { + SDObject &first = *o.data.children[0]; + + CHECK(first.name == "first"); + CHECK(first.type.name == "float"); + CHECK(first.type.basetype == SDBasic::Float); + CHECK(first.type.byteSize == 4); + CHECK(first.type.flags == SDTypeFlags::NoFlags); + + CHECK(first.data.basic.d == 3.14159f); + } + + { + SDObject &second = *o.data.children[1]; + + CHECK(second.name == "second"); + CHECK(second.type.name == "string"); + CHECK(second.type.basetype == SDBasic::String); + CHECK(second.type.byteSize == 4); + CHECK(second.type.flags == SDTypeFlags::NoFlags); + + CHECK(second.data.str == "M_PI"); + } + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "l"); + CHECK(o.type.basetype == SDBasic::Array); + CHECK(o.type.byteSize == 6); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + CHECK(o.data.children.size() == 6); + + for(SDObject *child : o.data.children) + { + CHECK(child->type.basetype == SDBasic::UnsignedInteger); + CHECK(child->type.byteSize == 2); + } + + CHECK(o.data.children[0]->data.basic.u == 2U); + CHECK(o.data.children[1]->data.basic.u == 3U); + CHECK(o.data.children[2]->data.basic.u == 5U); + CHECK(o.data.children[3]->data.basic.u == 7U); + CHECK(o.data.children[4]->data.basic.u == 11U); + CHECK(o.data.children[5]->data.basic.u == 13U); + } + + StreamWriter *rewriteBuf = new StreamWriter(StreamWriter::DefaultScratchSize); + + { + WriteSerialiser rewrite(rewriteBuf, Ownership::Nothing); + + rewrite.WriteStructuredFile(structData); + } + + // must be bitwise identical to the original serialised data. + REQUIRE(rewriteBuf->GetOffset() == buf->GetOffset()); + CHECK_FALSE(memcmp(rewriteBuf->GetData(), buf->GetData(), (size_t)rewriteBuf->GetOffset())); + + delete rewriteBuf; + } + + delete buf; +}; + +struct struct1 +{ + struct1() : x(0.0f), y(0.0f), width(0.0f), height(0.0f) {} + struct1(float X, float Y, float W, float H) : x(X), y(Y), width(W), height(H) {} + float x, y, width, height; +}; + +DECLARE_REFLECTION_STRUCT(struct1); + +template +void DoSerialise(SerialiserType &ser, struct1 &el) +{ + SERIALISE_MEMBER(x); + SERIALISE_MEMBER(y); + SERIALISE_MEMBER(width); + SERIALISE_MEMBER(height); +} + +struct struct2 +{ + string name; + std::vector floats; + std::vector viewports; +}; + +DECLARE_REFLECTION_STRUCT(struct2); + +template +void DoSerialise(SerialiserType &ser, struct2 &el) +{ + SERIALISE_MEMBER(name); + SERIALISE_MEMBER(floats); + SERIALISE_MEMBER(viewports); +} + +enum MySpecialEnum +{ + FirstEnumValue, + SecondEnumValue, + AnotherEnumValue, + TheLastEnumValue, +}; + +DECLARE_REFLECTION_ENUM(MySpecialEnum); + +template <> +std::string DoStringise(const MySpecialEnum &el) +{ + BEGIN_ENUM_STRINGISE(MySpecialEnum); + { + STRINGISE_ENUM(FirstEnumValue); + STRINGISE_ENUM(SecondEnumValue); + STRINGISE_ENUM(AnotherEnumValue); + STRINGISE_ENUM(TheLastEnumValue); + } + END_ENUM_STRINGISE(); +} + +TEST_CASE("Read/write complex types", "[serialiser][structured]") +{ + StreamWriter *buf = new StreamWriter(StreamWriter::DefaultScratchSize); + + { + WriteSerialiser ser(buf, Ownership::Nothing); + + SCOPED_SERIALISE_CHUNK(5); + + MySpecialEnum enumVal = AnotherEnumValue; + + SERIALISE_ELEMENT(enumVal); + + std::vector sparseStructArray; + + sparseStructArray.resize(10); + + sparseStructArray[5] = struct1(1.0f, 2.0f, 3.0f, 4.0f); + sparseStructArray[8] = struct1(10.0f, 20.0f, 30.0f, 40.0f); + + SERIALISE_ELEMENT(sparseStructArray); + + struct2 complex; + complex.name = "A complex object"; + complex.floats = {1.2f, 3.4f, 5.6f}; + complex.viewports.resize(4); + complex.viewports[0] = struct1(512.0f, 0.0f, 256.0f, 256.0f); + + SERIALISE_ELEMENT(complex); + + const struct1 *inputParam1 = new struct1(9.0f, 9.9f, 9.99f, 9.999f); + const struct1 *inputParam2 = NULL; + + SERIALISE_ELEMENT_OPT(inputParam1); + SERIALISE_ELEMENT_OPT(inputParam2); + + CHECK(buf->GetOffset() <= 512); + + REQUIRE_FALSE(ser.IsErrored()); + + delete inputParam1; + } + + { + ReadSerialiser ser(new StreamReader(buf->GetData(), buf->GetOffset()), Ownership::Stream); + + uint32_t chunkID = ser.ReadChunk(); + + CHECK(chunkID == 5); + + MySpecialEnum enumVal; + + SERIALISE_ELEMENT(enumVal); + + std::vector sparseStructArray; + + SERIALISE_ELEMENT(sparseStructArray); + + struct2 complex; + + SERIALISE_ELEMENT(complex); + + const struct1 *inputParam1; + const struct1 *inputParam2; + + SERIALISE_ELEMENT_OPT(inputParam1); + SERIALISE_ELEMENT_OPT(inputParam2); + + ser.EndChunk(); + + REQUIRE_FALSE(ser.IsErrored()); + + CHECK(ser.GetReader()->AtEnd()); + + CHECK(enumVal == AnotherEnumValue); + + CHECK(sparseStructArray[0].x == 0.0f); + CHECK(sparseStructArray[0].y == 0.0f); + CHECK(sparseStructArray[0].width == 0.0f); + CHECK(sparseStructArray[0].height == 0.0f); + + CHECK(sparseStructArray[5].x == 1.0f); + CHECK(sparseStructArray[5].y == 2.0f); + CHECK(sparseStructArray[5].width == 3.0f); + CHECK(sparseStructArray[5].height == 4.0f); + + CHECK(sparseStructArray[8].x == 10.0f); + CHECK(sparseStructArray[8].y == 20.0f); + CHECK(sparseStructArray[8].width == 30.0f); + CHECK(sparseStructArray[8].height == 40.0f); + + CHECK(complex.name == "A complex object"); + REQUIRE(complex.floats.size() == 3); + CHECK(complex.floats[0] == 1.2f); + CHECK(complex.floats[1] == 3.4f); + CHECK(complex.floats[2] == 5.6f); + REQUIRE(complex.viewports.size() == 4); + + CHECK(complex.viewports[0].x == 512.0f); + CHECK(complex.viewports[0].y == 0.0f); + CHECK(complex.viewports[0].width == 256.0f); + CHECK(complex.viewports[0].height == 256.0f); + + CHECK(inputParam1->x == 9.0f); + CHECK(inputParam1->y == 9.9f); + CHECK(inputParam1->width == 9.99f); + CHECK(inputParam1->height == 9.999f); + + CHECK(inputParam2 == NULL); + } + + { + ReadSerialiser ser(new StreamReader(buf->GetData(), buf->GetOffset()), Ownership::Stream); + + ser.ConfigureStructuredExport([](uint32_t) -> std::string { return "TestChunk"; }, true); + + ser.ReadChunk(); + { + MySpecialEnum enumVal; + + SERIALISE_ELEMENT(enumVal); + + std::vector sparseStructArray; + + SERIALISE_ELEMENT(sparseStructArray); + + struct2 complex; + + SERIALISE_ELEMENT(complex); + + const struct1 *inputParam1; + const struct1 *inputParam2; + + SERIALISE_ELEMENT_OPT(inputParam1); + SERIALISE_ELEMENT_OPT(inputParam2); + } + ser.EndChunk(); + + REQUIRE_FALSE(ser.IsErrored()); + + CHECK(ser.GetReader()->AtEnd()); + + const SDFile &structData = ser.GetStructuredFile(); + + CHECK(structData.chunks.size() == 1); + CHECK(structData.buffers.size() == 0); + + REQUIRE(structData.chunks[0]); + + const SDChunk &chunk = *structData.chunks[0]; + + CHECK(chunk.data.children.size() == 5); + + for(SDObject *o : chunk.data.children) + REQUIRE(o); + + int childIdx = 0; + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "enumVal"); + CHECK(o.type.basetype == SDBasic::Enum); + CHECK(o.type.flags == SDTypeFlags::HasCustomString); + + CHECK(o.data.basic.u == AnotherEnumValue); + CHECK(o.data.str == "AnotherEnumValue"); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "sparseStructArray"); + CHECK(o.type.basetype == SDBasic::Array); + CHECK(o.type.byteSize == 10); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + CHECK(o.data.children.size() == 10); + + for(SDObject *child : o.data.children) + { + CHECK(child->type.basetype == SDBasic::Struct); + CHECK(child->type.name == "struct1"); + CHECK(child->type.byteSize == sizeof(struct1)); + CHECK(child->data.children.size() == 4); + CHECK(child->data.children[0]->type.basetype == SDBasic::Float); + CHECK(child->data.children[0]->type.byteSize == 4); + CHECK(child->data.children[0]->name == "x"); + CHECK(child->data.children[1]->type.basetype == SDBasic::Float); + CHECK(child->data.children[1]->type.byteSize == 4); + CHECK(child->data.children[1]->name == "y"); + CHECK(child->data.children[2]->type.basetype == SDBasic::Float); + CHECK(child->data.children[2]->type.byteSize == 4); + CHECK(child->data.children[2]->name == "width"); + CHECK(child->data.children[3]->type.basetype == SDBasic::Float); + CHECK(child->data.children[3]->type.byteSize == 4); + CHECK(child->data.children[3]->name == "height"); + } + + CHECK(o.data.children[0]->data.children[0]->data.basic.d == 0.0f); + CHECK(o.data.children[0]->data.children[1]->data.basic.d == 0.0f); + CHECK(o.data.children[0]->data.children[2]->data.basic.d == 0.0f); + CHECK(o.data.children[0]->data.children[3]->data.basic.d == 0.0f); + + CHECK(o.data.children[5]->data.children[0]->data.basic.d == 1.0f); + CHECK(o.data.children[5]->data.children[1]->data.basic.d == 2.0f); + CHECK(o.data.children[5]->data.children[2]->data.basic.d == 3.0f); + CHECK(o.data.children[5]->data.children[3]->data.basic.d == 4.0f); + + CHECK(o.data.children[8]->data.children[0]->data.basic.d == 10.0f); + CHECK(o.data.children[8]->data.children[1]->data.basic.d == 20.0f); + CHECK(o.data.children[8]->data.children[2]->data.basic.d == 30.0f); + CHECK(o.data.children[8]->data.children[3]->data.basic.d == 40.0f); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "complex"); + CHECK(o.type.name == "struct2"); + CHECK(o.type.basetype == SDBasic::Struct); + CHECK(o.type.byteSize == sizeof(struct2)); + CHECK(o.type.flags == SDTypeFlags::NoFlags); + CHECK(o.data.children.size() == 3); + + { + SDObject &c = *o.data.children[0]; + + CHECK(c.name == "name"); + CHECK(c.type.name == "string"); + CHECK(c.type.basetype == SDBasic::String); + CHECK(c.type.flags == SDTypeFlags::NoFlags); + + CHECK(c.data.str == "A complex object"); + } + + { + SDObject &c = *o.data.children[1]; + + CHECK(c.name == "floats"); + CHECK(c.type.basetype == SDBasic::Array); + CHECK(c.type.flags == SDTypeFlags::NoFlags); + CHECK(c.data.children.size() == 3); + for(SDObject *ch : c.data.children) + { + CHECK(ch->type.basetype == SDBasic::Float); + CHECK(ch->type.byteSize == 4); + } + + CHECK(c.data.children[0]->data.basic.d == 1.2f); + CHECK(c.data.children[1]->data.basic.d == 3.4f); + CHECK(c.data.children[2]->data.basic.d == 5.6f); + } + + { + SDObject &c = *o.data.children[2]; + + CHECK(c.name == "viewports"); + CHECK(c.type.basetype == SDBasic::Array); + CHECK(c.type.flags == SDTypeFlags::NoFlags); + CHECK(c.data.children.size() == 4); + for(SDObject *ch : c.data.children) + { + CHECK(ch->type.basetype == SDBasic::Struct); + CHECK(ch->type.name == "struct1"); + } + + CHECK(c.data.children[0]->data.children[0]->data.basic.d == 512.0f); + CHECK(c.data.children[0]->data.children[1]->data.basic.d == 0.0f); + CHECK(c.data.children[0]->data.children[2]->data.basic.d == 256.0f); + CHECK(c.data.children[0]->data.children[3]->data.basic.d == 256.0f); + } + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "inputParam1"); + CHECK(o.type.basetype == SDBasic::Struct); + CHECK(o.type.flags == SDTypeFlags::Nullable); + + CHECK(o.data.children[0]->data.basic.d == 9.0f); + CHECK(o.data.children[1]->data.basic.d == 9.9f); + CHECK(o.data.children[2]->data.basic.d == 9.99f); + CHECK(o.data.children[3]->data.basic.d == 9.999f); + } + + { + SDObject &o = *chunk.data.children[childIdx++]; + + CHECK(o.name == "inputParam2"); + CHECK(o.type.basetype == SDBasic::Null); + CHECK(o.type.flags == SDTypeFlags::Nullable); + } + + StreamWriter *rewriteBuf = new StreamWriter(StreamWriter::DefaultScratchSize); + + { + WriteSerialiser rewrite(rewriteBuf, Ownership::Nothing); + + rewrite.WriteStructuredFile(structData); + } + + // must be bitwise identical to the original serialised data. + REQUIRE(rewriteBuf->GetOffset() == buf->GetOffset()); + CHECK_FALSE(memcmp(rewriteBuf->GetData(), buf->GetData(), (size_t)rewriteBuf->GetOffset())); + + delete rewriteBuf; + } + + delete buf; +}; + +#endif // ENABLED(ENABLE_UNIT_TESTS)