From 219bd2d4340e3a6cf8cc7be7d50938b3ed18268b Mon Sep 17 00:00:00 2001 From: baldurk Date: Sun, 13 Sep 2015 12:37:17 +0200 Subject: [PATCH] Refactor serialisation format with new version * The main improvement is that instead of just callstack resolving DB and frame capture data, some arbtirary number of sections can be included, for e.g. adding notes or saving the bookmarks or whatever use the user wants. * Sections (namely the frame capture data) can now be (de)compressed on I/O, which leads to much smaller file sizes. * Also the default internal alignment has changed from 16 to 64 to be more future proof (I decided not to go with completely arbitrary alignment). * A path for loading previous version logs remains and will be removed at some point in the future, maybe a version or two down the line. --- renderdoc/common/common.h | 3 + renderdoc/driver/d3d11/d3d11_context_wrap.cpp | 6 - renderdoc/driver/d3d11/d3d11_device.cpp | 6 +- renderdoc/driver/d3d11/d3d11_device.h | 2 - renderdoc/driver/d3d11/d3d11_device_wrap.cpp | 10 - renderdoc/driver/d3d11/d3d11_manager.h | 4 +- renderdoc/driver/gl/gl_driver.cpp | 6 +- renderdoc/driver/gl/gl_driver.h | 2 - renderdoc/driver/gl/gl_resources.h | 6 +- .../driver/gl/wrappers/gl_buffer_funcs.cpp | 13 +- renderdoc/os/linux/linux_stringio.cpp | 2 + renderdoc/os/os_specific.h | 2 + renderdoc/os/win32/win32_shellext.cpp | 6 +- renderdoc/os/win32/win32_stringio.cpp | 2 + renderdoc/serialise/serialiser.cpp | 1306 +++++++++++++---- renderdoc/serialise/serialiser.h | 207 +-- 16 files changed, 1106 insertions(+), 477 deletions(-) diff --git a/renderdoc/common/common.h b/renderdoc/common/common.h index 1c10a7edc..e36cbb8b3 100644 --- a/renderdoc/common/common.h +++ b/renderdoc/common/common.h @@ -85,6 +85,9 @@ inline T AlignUp16(T x) { return (x+0xf) & (~0xf); } template inline T AlignUp(T x, T a) { return (x+(a-1)) & (~(a-1)); } +template +inline T AlignUpPtr(T x, A a) { return (T)AlignUp( (uintptr_t)x, (uintptr_t)a ); } + #define MAKE_FOURCC(a, b, c, d) (((uint32_t)(d) << 24) | ((uint32_t)(c) << 16) | ((uint32_t)(b) << 8) | (uint32_t)(a)) bool FindDiffRange(void *a, void *b, size_t bufSize, size_t &diffStart, size_t &diffEnd); diff --git a/renderdoc/driver/d3d11/d3d11_context_wrap.cpp b/renderdoc/driver/d3d11/d3d11_context_wrap.cpp index 0c41c188a..86018cb0a 100644 --- a/renderdoc/driver/d3d11/d3d11_context_wrap.cpp +++ b/renderdoc/driver/d3d11/d3d11_context_wrap.cpp @@ -5182,8 +5182,6 @@ void WrappedID3D11DeviceContext::UpdateSubresource(ID3D11Resource *pDstResource, Serialise_UpdateSubresource(pDstResource, DstSubresource, pDstBox, pSrcData, SrcRowPitch, SrcDepthPitch); - scope.SetAlignment(32); - Chunk *chunk = scope.Get(); record->AddChunk(chunk); @@ -7133,8 +7131,6 @@ void WrappedID3D11DeviceContext::Unmap(ID3D11Resource *pResource, UINT Subresour m_pSerialiser->Serialise("context", m_ResourceID); Serialise_Unmap(pResource, Subresource); - scope.SetAlignment(32); - m_ContextRecord->AddChunk(scope.Get()); } else if(m_State >= WRITING) @@ -7162,8 +7158,6 @@ void WrappedID3D11DeviceContext::Unmap(ID3D11Resource *pResource, UINT Subresour m_pSerialiser->Serialise("context", m_ResourceID); Serialise_Unmap(pResource, Subresource); - scope.SetAlignment(32); - Chunk *chunk = scope.Get(); record->AddChunk(chunk); diff --git a/renderdoc/driver/d3d11/d3d11_device.cpp b/renderdoc/driver/d3d11/d3d11_device.cpp index f9ca6b0e4..5c438d941 100644 --- a/renderdoc/driver/d3d11/d3d11_device.cpp +++ b/renderdoc/driver/d3d11/d3d11_device.cpp @@ -295,8 +295,6 @@ WrappedID3D11Device::WrappedID3D11Device(ID3D11Device* realDevice, D3D11InitPara m_AppControlledCapture = false; m_TotalTime = m_AvgFrametime = m_MinFrametime = m_MaxFrametime = 0.0; - - m_CurFileSize = 0; #if defined(RELEASE) const bool debugSerialiser = false; @@ -1106,7 +1104,7 @@ void WrappedID3D11Device::ReadLogInitialisation() m_pSerialiser->SetDebugText(false); - m_pSerialiser->SetBase(firstFrame); + m_pSerialiser->SetPersistentBlock(firstFrame); } bool WrappedID3D11Device::Prepare_InitialState(ID3D11DeviceChild *res) @@ -2834,7 +2832,7 @@ bool WrappedID3D11Device::EndFrameCapture(void *dev, void *wnd) RDCDEBUG("Done"); } - m_CurFileSize += m_pFileSerialiser->FlushToDisk(); + m_pFileSerialiser->FlushToDisk(); SAFE_DELETE(m_pFileSerialiser); diff --git a/renderdoc/driver/d3d11/d3d11_device.h b/renderdoc/driver/d3d11/d3d11_device.h index 23c656a8a..0c76c7d0e 100644 --- a/renderdoc/driver/d3d11/d3d11_device.h +++ b/renderdoc/driver/d3d11/d3d11_device.h @@ -261,8 +261,6 @@ private: CaptureFailReason m_FailedReason; uint32_t m_Failures; - uint64_t m_CurFileSize; - PerformanceTimer m_FrameTimer; vector m_FrameTimes; double m_TotalTime, m_AvgFrametime, m_MinFrametime, m_MaxFrametime; diff --git a/renderdoc/driver/d3d11/d3d11_device_wrap.cpp b/renderdoc/driver/d3d11/d3d11_device_wrap.cpp index c0f32ae7c..7eaa83fbd 100644 --- a/renderdoc/driver/d3d11/d3d11_device_wrap.cpp +++ b/renderdoc/driver/d3d11/d3d11_device_wrap.cpp @@ -165,8 +165,6 @@ HRESULT WrappedID3D11Device::CreateBuffer( SCOPED_SERIALISE_CONTEXT(CREATE_BUFFER); Serialise_CreateBuffer(pDesc, pInitialData, &wrapped); - scope.SetAlignment(32); - chunk = scope.Get(); } @@ -400,8 +398,6 @@ HRESULT WrappedID3D11Device::CreateTexture1D( SCOPED_SERIALISE_CONTEXT(CREATE_TEXTURE_1D); Serialise_CreateTexture1D(pDesc, pInitialData, &wrapped); - scope.SetAlignment(32); - chunk = scope.Get(); } @@ -503,8 +499,6 @@ HRESULT WrappedID3D11Device::CreateTexture2D( SCOPED_SERIALISE_CONTEXT(CREATE_TEXTURE_2D); Serialise_CreateTexture2D(pDesc, pInitialData, &wrapped); - scope.SetAlignment(32); - chunk = scope.Get(); } @@ -606,8 +600,6 @@ HRESULT WrappedID3D11Device::CreateTexture3D( SCOPED_SERIALISE_CONTEXT(CREATE_TEXTURE_3D); Serialise_CreateTexture3D(pDesc, pInitialData, &wrapped); - scope.SetAlignment(32); - chunk = scope.Get(); } @@ -2842,8 +2834,6 @@ HRESULT WrappedID3D11Device::OpenSharedResource( SCOPED_SERIALISE_CONTEXT(OPEN_SHARED_RESOURCE); Serialise_OpenSharedResource(hResource, ReturnedInterface, ppResource); - scope.SetAlignment(32); - chunk = scope.Get(); } diff --git a/renderdoc/driver/d3d11/d3d11_manager.h b/renderdoc/driver/d3d11/d3d11_manager.h index 7804e58b8..d3946e7b2 100644 --- a/renderdoc/driver/d3d11/d3d11_manager.h +++ b/renderdoc/driver/d3d11/d3d11_manager.h @@ -62,8 +62,8 @@ struct D3D11ResourceRecord : public ResourceRecord { if(ShadowPtr[ctx][0] == NULL) { - ShadowPtr[ctx][0] = Serialiser::AllocAlignedBuffer(size + sizeof(markerValue), 32); - ShadowPtr[ctx][1] = Serialiser::AllocAlignedBuffer(size + sizeof(markerValue), 32); + ShadowPtr[ctx][0] = Serialiser::AllocAlignedBuffer(size + sizeof(markerValue)); + ShadowPtr[ctx][1] = Serialiser::AllocAlignedBuffer(size + sizeof(markerValue)); memcpy(ShadowPtr[ctx][0] + size, markerValue, sizeof(markerValue)); memcpy(ShadowPtr[ctx][1] + size, markerValue, sizeof(markerValue)); diff --git a/renderdoc/driver/gl/gl_driver.cpp b/renderdoc/driver/gl/gl_driver.cpp index 47b35ec49..a5f6013ac 100644 --- a/renderdoc/driver/gl/gl_driver.cpp +++ b/renderdoc/driver/gl/gl_driver.cpp @@ -702,8 +702,6 @@ WrappedOpenGL::WrappedOpenGL(const char *logfile, const GLHookSet &funcs) m_TotalTime = m_AvgFrametime = m_MinFrametime = m_MaxFrametime = 0.0; - m_CurFileSize = 0; - m_RealDebugFunc = NULL; m_RealDebugFuncParam = NULL; @@ -2434,7 +2432,7 @@ bool WrappedOpenGL::EndFrameCapture(void *dev, void *wnd) RDCDEBUG("Done"); } - m_CurFileSize += m_pFileSerialiser->FlushToDisk(); + m_pFileSerialiser->FlushToDisk(); RenderDoc::Inst().SuccessfullyWrittenLog(); @@ -2953,7 +2951,7 @@ void WrappedOpenGL::ReadLogInitialisation() m_pSerialiser->SetDebugText(false); - m_pSerialiser->SetBase(firstFrame); + m_pSerialiser->SetPersistentBlock(firstFrame); } void WrappedOpenGL::ProcessChunk(uint64_t offset, GLChunkType context) diff --git a/renderdoc/driver/gl/gl_driver.h b/renderdoc/driver/gl/gl_driver.h index 05874caf7..7316c1f36 100644 --- a/renderdoc/driver/gl/gl_driver.h +++ b/renderdoc/driver/gl/gl_driver.h @@ -164,8 +164,6 @@ class WrappedOpenGL : public IFrameCapturer CaptureFailReason m_FailureReason; bool m_SuccessfulCapture; - uint64_t m_CurFileSize; - PerformanceTimer m_FrameTimer; vector m_FrameTimes; double m_TotalTime, m_AvgFrametime, m_MinFrametime, m_MaxFrametime; diff --git a/renderdoc/driver/gl/gl_resources.h b/renderdoc/driver/gl/gl_resources.h index af66b1088..df5ef2d51 100644 --- a/renderdoc/driver/gl/gl_resources.h +++ b/renderdoc/driver/gl/gl_resources.h @@ -184,12 +184,12 @@ struct GLResourceRecord : public ResourceRecord GLResource Resource; - void AllocShadowStorage(size_t size, size_t alignment = 16) + void AllocShadowStorage(size_t size) { if(ShadowPtr[0] == NULL) { - ShadowPtr[0] = Serialiser::AllocAlignedBuffer(size, alignment); - ShadowPtr[1] = Serialiser::AllocAlignedBuffer(size, alignment); + ShadowPtr[0] = Serialiser::AllocAlignedBuffer(size); + ShadowPtr[1] = Serialiser::AllocAlignedBuffer(size); } } diff --git a/renderdoc/driver/gl/wrappers/gl_buffer_funcs.cpp b/renderdoc/driver/gl/wrappers/gl_buffer_funcs.cpp index f8f893ef1..3190f384a 100644 --- a/renderdoc/driver/gl/wrappers/gl_buffer_funcs.cpp +++ b/renderdoc/driver/gl/wrappers/gl_buffer_funcs.cpp @@ -344,9 +344,6 @@ void WrappedOpenGL::Common_glNamedBufferStorageEXT(ResourceId id, GLsizeiptr siz SCOPED_SERIALISE_CONTEXT(BUFFERSTORAGE); Serialise_glNamedBufferStorageEXT(record->Resource.name, size, data, flags); - // for satisfying GL_MIN_MAP_BUFFER_ALIGNMENT - scope.SetAlignment(64); - Chunk *chunk = scope.Get(); { @@ -368,7 +365,7 @@ void WrappedOpenGL::Common_glNamedBufferStorageEXT(ResourceId id, GLsizeiptr siz RDCASSERT(record->Map.persistentPtr); // persistent maps always need both sets of shadow storage, so allocate up front. - record->AllocShadowStorage(size, 64); + record->AllocShadowStorage(size); } } else @@ -544,9 +541,6 @@ void WrappedOpenGL::glNamedBufferDataEXT(GLuint buffer, GLsizeiptr size, const v SCOPED_SERIALISE_CONTEXT(BUFFERDATA); Serialise_glNamedBufferDataEXT(buffer, size, data, usage); - // for satisfying GL_MIN_MAP_BUFFER_ALIGNMENT - scope.SetAlignment(64); - Chunk *chunk = scope.Get(); // if we've already created this is a renaming/data updating call. It should go in @@ -675,9 +669,6 @@ void WrappedOpenGL::glBufferData(GLenum target, GLsizeiptr size, const void *dat SCOPED_SERIALISE_CONTEXT(BUFFERDATA); Serialise_glNamedBufferDataEXT(buffer, size, data, usage); - // for satisfying GL_MIN_MAP_BUFFER_ALIGNMENT - scope.SetAlignment(64); - Chunk *chunk = scope.Get(); // if we've already created this is a renaming/data updating call. It should go in @@ -1752,7 +1743,7 @@ void *WrappedOpenGL::glMapNamedBufferRangeEXT(GLuint buffer, GLintptr offset, GL m_Real.glGetNamedBufferParameterivEXT(buffer, eGL_BUFFER_SIZE, &buflength); // allocate our shadow storage - record->AllocShadowStorage(buflength, 64); + record->AllocShadowStorage(buflength); shadow = (byte *)record->GetShadowPtr(0); // if we're not invalidating, we need the existing contents diff --git a/renderdoc/os/linux/linux_stringio.cpp b/renderdoc/os/linux/linux_stringio.cpp index 006ed3d50..a4ec891e7 100644 --- a/renderdoc/os/linux/linux_stringio.cpp +++ b/renderdoc/os/linux/linux_stringio.cpp @@ -345,6 +345,8 @@ namespace FileIO uint64_t ftell64(FILE *f) { return (uint64_t)::ftell(f); } void fseek64(FILE *f, uint64_t offset, int origin) { ::fseek(f, (long)offset, origin); } + bool feof(FILE *f) { return ::feof(f) != 0; } + int fclose(FILE *f) { return ::fclose(f); } }; diff --git a/renderdoc/os/os_specific.h b/renderdoc/os/os_specific.h index fbbbffa54..7db01c1cb 100644 --- a/renderdoc/os/os_specific.h +++ b/renderdoc/os/os_specific.h @@ -200,6 +200,8 @@ namespace FileIO uint64_t ftell64(FILE *f); void fseek64(FILE *f, uint64_t offset, int origin); + bool feof(FILE *f); + int fclose(FILE *f); }; diff --git a/renderdoc/os/win32/win32_shellext.cpp b/renderdoc/os/win32/win32_shellext.cpp index 7675fe419..fd53cef59 100644 --- a/renderdoc/os/win32/win32_shellext.cpp +++ b/renderdoc/os/win32/win32_shellext.cpp @@ -108,9 +108,9 @@ struct RDCThumbnailProvider : public IThumbnailProvider, IInitializeWithStream if(m_Inited) return HRESULT_FROM_WIN32(ERROR_ALREADY_INITIALIZED); - byte *buf = new byte[512*1024 + 1]; + byte *buf = new byte[2*1024*1024 + 1]; ULONG numRead = 0; - HRESULT hr = pstream->Read(buf, 512*1024, &numRead); + HRESULT hr = pstream->Read(buf, 2*1024*1024, &numRead); if(hr != S_OK && hr != S_FALSE) { @@ -122,6 +122,8 @@ struct RDCThumbnailProvider : public IThumbnailProvider, IInitializeWithStream m_Ser = new Serialiser(numRead, buf, true); + delete[] buf; + m_Inited = true; return S_OK; diff --git a/renderdoc/os/win32/win32_stringio.cpp b/renderdoc/os/win32/win32_stringio.cpp index ad6e03f68..db637c280 100644 --- a/renderdoc/os/win32/win32_stringio.cpp +++ b/renderdoc/os/win32/win32_stringio.cpp @@ -337,6 +337,8 @@ namespace FileIO uint64_t ftell64(FILE *f) { return ::_ftelli64(f); } void fseek64(FILE *f, uint64_t offset, int origin) { ::_fseeki64(f, offset, origin); } + bool feof(FILE *f) { return ::feof(f) != 0; } + int fclose(FILE *f) { return ::fclose(f); } }; diff --git a/renderdoc/serialise/serialiser.cpp b/renderdoc/serialise/serialiser.cpp index 480363489..0ce130c8d 100644 --- a/renderdoc/serialise/serialiser.cpp +++ b/renderdoc/serialise/serialiser.cpp @@ -28,6 +28,9 @@ #include "core/core.h" #include "serialise/string_utils.h" +#include "common/timing.h" + +#include "3rdparty/lz4/lz4.h" #ifdef _MSC_VER #pragma warning (disable : 4422) // warning C4422: 'snprintf' : too many arguments passed for format string @@ -43,9 +46,198 @@ int64_t Chunk::m_MaxChunks = 0; #endif const uint32_t Serialiser::MAGIC_HEADER = MAKE_FOURCC('R', 'D', 'O', 'C'); -const size_t Serialiser::BufferAlignment = 16; +const uint64_t Serialiser::BufferAlignment = 64; -Chunk::Chunk(Serialiser *ser, uint32_t chunkType, size_t alignment, bool temporary) +// based on blockStreaming_doubleBuffer.c in lz4 examples +struct CompressedFileIO +{ + // large block size + static const size_t BlockSize = 64 * 1024; + + CompressedFileIO(FILE *f) + { + m_F = f; + LZ4_resetStream(&m_LZ4Comp); + LZ4_setStreamDecode(&m_LZ4Decomp, NULL, 0); + m_CompressedSize = m_UncompressedSize = 0; + m_PageIdx = m_PageOffset = 0; + m_PageData = 0; + + m_CompressSize = LZ4_COMPRESSBOUND(BlockSize); + m_CompressBuf = new byte[m_CompressSize]; + } + + ~CompressedFileIO() + { + SAFE_DELETE_ARRAY(m_CompressBuf); + } + + uint32_t GetCompressedSize() { return m_CompressedSize; } + uint32_t GetUncompressedSize() { return m_UncompressedSize; } + + // write out some data - accumulate into the input pages, then + // when a page is full call Flush() to flush it out to disk + void Write(const void *data, size_t len) + { + if(data == NULL || len == 0) return; + + m_UncompressedSize += (uint32_t)len; + + const byte *src = (const byte *)data; + + size_t remainder = 0; + + // loop continually, writing up to BlockSize out of what remains of data + do + { + remainder = 0; + + // if we're about to copy more than the page, copy only + // what will fit, then copy the remainder after flushing + if(m_PageOffset + len > BlockSize) + { + remainder = len - (BlockSize - m_PageOffset); + len = BlockSize - m_PageOffset; + } + + memcpy(m_InPages[m_PageIdx] + m_PageOffset, src, len); + m_PageOffset += len; + + if(remainder > 0) + { + Flush(); // this will swap the input pages and reset the page offset + + src += len; + len = remainder; + } + } while(remainder > 0); + } + + // flush out the current page to disk + void Flush() + { + // m_PageOffset is the amount written, usually equal to BlockSize except the last block. + int32_t compSize = LZ4_compress_fast_continue(&m_LZ4Comp, (const char *)m_InPages[m_PageIdx], (char *)m_CompressBuf, (int)m_PageOffset, (int)m_CompressSize, 1); + + if(compSize < 0) + { + RDCERR("Error compressing: %i", compSize); + return; + } + + FileIO::fwrite(&compSize, sizeof(compSize), 1, m_F); + FileIO::fwrite(m_CompressBuf, 1, compSize, m_F); + + m_CompressedSize += compSize + sizeof(int32_t); + + m_PageOffset = 0; + m_PageIdx = 1 - m_PageIdx; + } + + // Reset back to 0, only makes sense when reading as writing can't be undone + void Reset() + { + LZ4_setStreamDecode(&m_LZ4Decomp, NULL, 0); + m_CompressedSize = m_UncompressedSize = 0; + m_PageIdx = 0; + m_PageOffset = 0; + } + + // read out some data - if the input page is empty we fill + // the next page with data from disk + void Read(byte *data, size_t len) + { + if(data == NULL || len == 0) return; + + m_UncompressedSize += (uint32_t)len; + + // loop continually, writing up to BlockSize out of what remains of data + do + { + size_t readamount = len; + + // if we're about to copy more than the page, copy only + // what will fit, then copy the remainder after refilling + if(readamount > m_PageData) + readamount = m_PageData; + + if(readamount > 0) + { + memcpy(data, m_InPages[m_PageIdx] + m_PageOffset, readamount); + + m_PageOffset += readamount; + m_PageData -= readamount; + + data += readamount; + len -= readamount; + } + + if(len > 0) + FillBuffer(); // this will swap the input pages and reset the page offset + } while(len > 0); + } + + void FillBuffer() + { + int32_t compSize = 0; + + FileIO::fread(&compSize, sizeof(compSize), 1, m_F); + FileIO::fread(m_CompressBuf, 1, compSize, m_F); + + m_CompressedSize += compSize; + + m_PageIdx = 1 - m_PageIdx; + + int32_t decompSize = LZ4_decompress_safe_continue(&m_LZ4Decomp, (const char *)m_CompressBuf, (char *)m_InPages[m_PageIdx], compSize, BlockSize); + + if(decompSize < 0) + { + RDCERR("Error decompressing: %i", decompSize); + return; + } + + m_PageOffset = 0; + m_PageData = decompSize; + } + + static void Decompress(byte *destBuf, const byte *srcBuf, size_t len) + { + LZ4_streamDecode_t lz4; + LZ4_setStreamDecode(&lz4, NULL, 0); + + const byte *srcBufEnd = srcBuf + len; + + while(srcBuf+4 < srcBufEnd) + { + const int32_t *compSize = (const int32_t *)srcBuf; + srcBuf = (const byte *)(compSize + 1); + + if(srcBuf + *compSize > srcBufEnd) + break; + + int32_t decompSize = LZ4_decompress_safe_continue(&lz4, (const char *)srcBuf, (char *)destBuf, *compSize, BlockSize); + + if(decompSize < 0) + return; + + srcBuf += *compSize; + destBuf += decompSize; + } + } + + LZ4_stream_t m_LZ4Comp; + LZ4_streamDecode_t m_LZ4Decomp; + FILE *m_F; + uint32_t m_CompressedSize, m_UncompressedSize; + + byte m_InPages[2][BlockSize]; + size_t m_PageIdx, m_PageOffset, m_PageData; + + byte *m_CompressBuf; + size_t m_CompressSize; +}; + +Chunk::Chunk(Serialiser *ser, uint32_t chunkType, bool temporary) { m_Length = (uint32_t)ser->GetOffset(); @@ -55,12 +247,7 @@ Chunk::Chunk(Serialiser *ser, uint32_t chunkType, size_t alignment, bool tempora m_Temporary = temporary; - if(alignment) - { - m_Data = Serialiser::AllocAlignedBuffer(m_Length, alignment); - m_AlignedData = true; - } - else if(ser->HasAlignedData()) + if(ser->HasAlignedData()) { m_Data = Serialiser::AllocAlignedBuffer(m_Length); m_AlignedData = true; @@ -110,77 +297,132 @@ Chunk::~Chunk() } } -void Serialiser::Reset() +/* + + ----------------------------- + File format for version 0x31: + + uint64_t MAGIC_HEADER; + uint64_t version = 0x00000031; + uint64_t filesize; + uint64_t resolveDBSize; + + if(resolveDBSize > 0) + { + byte resolveDB[resolveDBSize]; + byte paddingTo16Alignment[]; + } + + byte captureData[]; // remainder of the file + + ----------------------------- + File format for version 0x32: + + uint64_t MAGIC_HEADER; + uint64_t version = 0x00000032; + + 1 or more sections: + + Section + { + char isASCII = '\0' or 'A'; // indicates the section is ASCII or binary. ASCII allows for easy appending by hand/script + if(isASCII == 'A') + { + // ASCII sections are discouraged for tools, but useful for hand-editing by just + // appending a simple text file + char newline = '\n'; + char length[]; // length of just section data below, as decimal string + char newline = '\n'; + char sectionType[]; // section type, see SectionType enum, as decimal string. + char newline = '\n'; + char sectionName[]; // UTF-8 string name of section. + char newline = '\n'; + // sectionName is an arbitrary string. If two sections exist with the + // exact same name, the last one to occur in the file will be taken, the + // others ignored. + + byte sectiondata[ interpret(length) ]; // section data + } + else if(isASCII == '\0') + { + byte zero[3]; // pad out the above character with 0 bytes. Reserved for future use + uint32_t sectionFlags; // section flags - e.g. is compressed or not. + uint32_t sectionType; // section type enum, see SectionType. Could be eSectionType_Unknown + uint32_t sectionLength; // byte length of the actual section data + uint32_t sectionNameLength; // byte length of the string below (minimum 1, for null terminator) + char sectionName[sectionNameLength]; // UTF-8 string name of section, optional. + + byte sectiondata[length]; // actual contents of the section + + // note: compressed sections will contain the uncompressed length as a uint64_t + // before the compressed data. + } + }; + + // remainder of the file is tightly packed/unaligned section structures. + // The first section must always be the actual frame capture data in + // binary form + Section sections[]; + +*/ + +struct FileHeader { - if(m_ResolverThread != 0) + FileHeader() { - m_ResolverThreadKillSignal = true; - - Threading::JoinThread(m_ResolverThread); - Threading::CloseThread(m_ResolverThread); - m_ResolverThread = 0; + magic = Serialiser::MAGIC_HEADER; + version = Serialiser::SERIALISE_VERSION; } - m_DebugText = ""; + uint64_t magic; + uint64_t version; +}; - m_HasError = false; - m_ErrorCode = eSerError_None; +struct BinarySectionHeader +{ + byte isASCII; // 0x0 + byte zero[3]; // 0x0, 0x0, 0x0 + Serialiser::SectionFlags sectionFlags; // section flags - e.g. is compressed or not. + Serialiser::SectionType sectionType; // section type enum, see SectionType. Could be eSectionType_Unknown + uint32_t sectionLength; // byte length of the actual section data + uint32_t sectionNameLength; // byte length of the string below (could be 0) + char name[1]; // actually sectionNameLength, but at least 1 for null terminator - m_Mode = NONE; + // char name[sectionNameLength]; + // byte data[sectionLength]; +}; - m_Indent = 0; - - SAFE_DELETE_ARRAY(m_pCallstack); - SAFE_DELETE(m_pResolver); - if(m_Buffer) - { - FreeAlignedBuffer(m_Buffer); - m_Buffer = NULL; - } - - m_ChunkLookup = NULL; - - m_HasResolver = false; - - m_AlignedData = false; - - m_ReadFileHandle = NULL; - - m_Buffer = NULL; - m_BufferSize = 0; - m_BufferHead = NULL; -} +#define RETURNCORRUPT(...) { RDCERR(__VA_ARGS__); m_ErrorCode = eSerError_Corrupt; m_HasError = true; return; } Serialiser::Serialiser(size_t length, const byte *memoryBuf, bool fileheader) : m_pCallstack(NULL), m_pResolver(NULL), m_Buffer(NULL) { m_ResolverThread = 0; + // ensure binary sizes of enums + RDCCOMPILE_ASSERT(sizeof(SectionFlags) == sizeof(uint32_t), "Section flags not in uint32"); + RDCCOMPILE_ASSERT(sizeof(SectionType) == sizeof(uint32_t), "Section type not in uint32"); + + RDCCOMPILE_ASSERT(offsetof(BinarySectionHeader, name) == sizeof(uint32_t)*5, "BinarySectionHeader size has changed or contains padding"); + Reset(); - - m_DebugTextWriting = false; m_Mode = READING; m_DebugEnabled = false; if(!fileheader) { - m_HasResolver = false; - - m_FileStartOffset = 0; - m_BufferSize = length; m_CurrentBufferSize = (size_t)m_BufferSize; m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); - m_ReadOffset = 0; - memcpy(m_Buffer, memoryBuf + m_FileStartOffset, m_CurrentBufferSize); + memcpy(m_Buffer, memoryBuf, m_CurrentBufferSize); return; } - DebuggerHeader *header = (DebuggerHeader *)memoryBuf; + FileHeader *header = (FileHeader *)memoryBuf; - if(length < sizeof(DebuggerHeader)) + if(length < sizeof(FileHeader)) { RDCERR("Can't read from in-memory buffer, truncated header"); m_ErrorCode = eSerError_Corrupt; @@ -201,7 +443,129 @@ Serialiser::Serialiser(size_t length, const byte *memoryBuf, bool fileheader) return; } - if(header->version != SERIALISE_VERSION) + const byte *memoryBufEnd = memoryBuf + length; + + m_SerVer = header->version; + + if(header->version == 0x00000031) // backwards compatibility + { + memoryBuf += sizeof(FileHeader); + + if(length < sizeof(FileHeader) + sizeof(uint64_t)*2) + { + RDCERR("Can't read from in-memory buffer, truncated header"); + m_ErrorCode = eSerError_Corrupt; + m_HasError = true; + return; + } + + uint64_t *fileSize = (uint64_t *)memoryBuf; + memoryBuf += sizeof(uint64_t); + + uint64_t *resolveDBSize = (uint64_t *)memoryBuf; + memoryBuf += sizeof(uint64_t); + + if(*fileSize < length) + { + RDCERR("Overlong in-memory buffer. Expected length 0x016llx, got 0x016llx", *fileSize, length); + + m_ErrorCode = eSerError_Corrupt; + m_HasError = true; + return; + } + + // for in-memory case we don't need to load up the resolve db + + Section *frameCap = new Section(); + frameCap->type = eSectionType_FrameCapture; + frameCap->flags = eSectionFlag_None; + frameCap->fileoffset = 0; // irrelevant + frameCap->name = "renderdoc/internal/framecapture"; + frameCap->size = uint64_t(memoryBufEnd - memoryBuf); + + memoryBuf = AlignUpPtr(memoryBuf + *resolveDBSize, 16); + + m_Sections.push_back(frameCap); + m_KnownSections[eSectionType_FrameCapture] = frameCap; + } + else if(header->version == SERIALISE_VERSION) + { + const byte *memoryBufEnd = memoryBuf + length; + + memoryBuf += sizeof(FileHeader); + + // when loading in-memory we only care about the first section, which should be binary + const BinarySectionHeader *sectionHeader = (const BinarySectionHeader *)memoryBuf; + + // verify validity + if(memoryBuf + offsetof(BinarySectionHeader, name) >= memoryBufEnd) + { + RDCERR("Truncated binary section header"); + + m_ErrorCode = eSerError_Corrupt; + m_HasError = true; + return; + } + + if(sectionHeader->isASCII != 0 || + sectionHeader->zero[0] != 0 || + sectionHeader->zero[1] != 0 || + sectionHeader->zero[2] != 0) + { + RDCERR("Unexpected non-binary section first in capture when loading in-memory"); + + m_ErrorCode = eSerError_Corrupt; + m_HasError = true; + return; + } + + if(sectionHeader->sectionType != eSectionType_FrameCapture) + { + RDCERR("Expected first section to be frame capture, got type %x", sectionHeader->sectionType); + + m_ErrorCode = eSerError_Corrupt; + m_HasError = true; + return; + } + + memoryBuf += offsetof(BinarySectionHeader, name); + memoryBuf += sectionHeader->sectionNameLength; // skip name + + if(memoryBuf >= memoryBufEnd) + { + RDCERR("Truncated binary section header"); + + m_ErrorCode = eSerError_Corrupt; + m_HasError = true; + return; + } + + Section *frameCap = new Section(); + frameCap->fileoffset = 0; // irrelevant + frameCap->data.assign(memoryBuf, memoryBufEnd); + frameCap->name = sectionHeader->name; + frameCap->type = sectionHeader->sectionType; + frameCap->flags = sectionHeader->sectionFlags; + + uint64_t *uncompLength = (uint64_t *)memoryBuf; + + memoryBuf += sizeof(uint64_t); + + if(memoryBuf >= memoryBufEnd) + { + RDCERR("Truncated binary section header"); + + m_ErrorCode = eSerError_Corrupt; + m_HasError = true; + return; + } + + frameCap->size = *uncompLength; + + m_KnownSections[eSectionType_FrameCapture] = frameCap; + m_Sections.push_back(frameCap); + } + else { RDCERR("Capture file from wrong version. This program is on logfile version %llu, file is logfile version %llu", SERIALISE_VERSION, header->version); @@ -210,25 +574,18 @@ Serialiser::Serialiser(size_t length, const byte *memoryBuf, bool fileheader) return; } - if(header->fileSize < length) - { - RDCERR("Overlong in-memory buffer. Expected length 0x016llx, got 0x016llx", header->fileSize, length); - - m_ErrorCode = eSerError_Corrupt; - m_HasError = true; - return; - } - - m_HasResolver = header->resolveDBSize > 0; - - m_FileStartOffset = AlignUp16(sizeof(DebuggerHeader) + header->resolveDBSize); - - m_BufferSize = length-m_FileStartOffset; + m_BufferSize = m_KnownSections[eSectionType_FrameCapture]->size; m_CurrentBufferSize = (size_t)m_BufferSize; m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); - m_ReadOffset = 0; - memcpy(m_Buffer, memoryBuf + m_FileStartOffset, m_CurrentBufferSize); + if(m_KnownSections[eSectionType_FrameCapture]->flags & eSectionFlag_LZ4Compressed) + { + CompressedFileIO::Decompress(m_Buffer, memoryBuf, memoryBufEnd - memoryBuf); + } + else + { + memcpy(m_Buffer, memoryBuf, m_CurrentBufferSize); + } } Serialiser::Serialiser(const char *path, Mode mode, bool debugMode) @@ -239,13 +596,11 @@ Serialiser::Serialiser(const char *path, Mode mode, bool debugMode) Reset(); m_Filename = path ? path : ""; - - m_DebugTextWriting = false; m_Mode = mode; m_DebugEnabled = debugMode; - DebuggerHeader header; + FileHeader header; if(mode == READING) { @@ -261,7 +616,7 @@ Serialiser::Serialiser(const char *path, Mode mode, bool debugMode) RDCDEBUG("Opened capture file for read"); - FileIO::fread(&header, 1, sizeof(DebuggerHeader), m_ReadFileHandle); + FileIO::fread(&header, 1, sizeof(FileHeader), m_ReadFileHandle); if(header.magic != MAGIC_HEADER) { @@ -273,8 +628,201 @@ Serialiser::Serialiser(const char *path, Mode mode, bool debugMode) m_ReadFileHandle = 0; return; } - - if(header.version != SERIALISE_VERSION) + + m_SerVer = header.version; + + if(header.version == 0x00000031) // backwards compatibility + { + uint64_t headerRemainder[2]; + + FileIO::fread(&headerRemainder, 1, sizeof(headerRemainder), m_ReadFileHandle); + + FileIO::fseek64(m_ReadFileHandle, 0, SEEK_END); + + uint64_t realLength = FileIO::ftell64(m_ReadFileHandle); + if(headerRemainder[0] != realLength) + { + RDCERR("Truncated/overlong capture file. Expected length 0x016llx, got 0x016llx", headerRemainder[0], realLength); + + m_ErrorCode = eSerError_Corrupt; + m_HasError = true; + FileIO::fclose(m_ReadFileHandle); + m_ReadFileHandle = 0; + return; + } + + FileIO::fseek64(m_ReadFileHandle, sizeof(FileHeader) + sizeof(headerRemainder), SEEK_SET); + + size_t resolveDBSize = (size_t)headerRemainder[1]; + + if(resolveDBSize > 0) + { + Section *resolveDB = new Section(); + resolveDB->type = eSectionType_ResolveDatabase; + resolveDB->flags = eSectionFlag_None; + resolveDB->fileoffset = sizeof(FileHeader) + sizeof(headerRemainder); + resolveDB->name = "renderdoc/internal/resolvedb"; + resolveDB->data.resize(resolveDBSize); + + // read resolve DB entirely into memory + FileIO::fread(&resolveDB->data[0], 1, resolveDBSize, m_ReadFileHandle); + + m_Sections.push_back(resolveDB); + m_KnownSections[eSectionType_ResolveDatabase] = resolveDB; + } + + // seek to frame capture data + FileIO::fseek64(m_ReadFileHandle, AlignUp16(sizeof(FileHeader) + sizeof(headerRemainder) + resolveDBSize), SEEK_SET); + + Section *frameCap = new Section(); + frameCap->type = eSectionType_FrameCapture; + frameCap->flags = eSectionFlag_None; + frameCap->fileoffset = FileIO::ftell64(m_ReadFileHandle); + frameCap->name = "renderdoc/internal/framecapture"; + frameCap->size = realLength - frameCap->fileoffset; + + m_Sections.push_back(frameCap); + m_KnownSections[eSectionType_FrameCapture] = frameCap; + } + else if(header.version == SERIALISE_VERSION) + { + while(!FileIO::feof(m_ReadFileHandle)) + { + BinarySectionHeader sectionHeader = {0}; + byte *reading = (byte *)§ionHeader; + + FileIO::fread(reading, 1, 1, m_ReadFileHandle); reading++; + + if(FileIO::feof(m_ReadFileHandle)) + break; + + if(sectionHeader.isASCII == 'A') + { + // ASCII section + char c = 0; + FileIO::fread(&c, 1, 1, m_ReadFileHandle); + if(c != '\n') + RETURNCORRUPT("Invalid ASCII data section '%hhx'", c); + + if(FileIO::feof(m_ReadFileHandle)) + RETURNCORRUPT("Invalid truncated ASCII data section"); + + uint64_t length = 0; + + c = '0'; + + while(!FileIO::feof(m_ReadFileHandle) && c != '\n') + { + c = '0'; + FileIO::fread(&c, 1, 1, m_ReadFileHandle); + + if(c == '\n') break; + + length *= 10; + length += int(c - '0'); + } + + if(FileIO::feof(m_ReadFileHandle)) RETURNCORRUPT("Invalid truncated ASCII data section"); + + union + { + uint32_t u32; + SectionType t; + } type; + + type.u32 = 0; + + c = '0'; + + while(!FileIO::feof(m_ReadFileHandle) && c != '\n') + { + c = '0'; + FileIO::fread(&c, 1, 1, m_ReadFileHandle); + + if(c == '\n') break; + + type.u32 *= 10; + type.u32 += int(c - '0'); + } + + if(FileIO::feof(m_ReadFileHandle)) RETURNCORRUPT("Invalid truncated ASCII data section"); + + string name; + + c = 0; + + while(!FileIO::feof(m_ReadFileHandle) && c != '\n') + { + c = 0; + FileIO::fread(&c, 1, 1, m_ReadFileHandle); + + if(c == 0 || c == '\n') break; + + name.push_back(c); + } + + if(FileIO::feof(m_ReadFileHandle)) RETURNCORRUPT("Invalid truncated ASCII data section"); + + Section *sect = new Section(); + sect->flags = eSectionFlag_ASCIIStored; + sect->type = type.t; + sect->name = name; + sect->size = length; + sect->data.resize((size_t)length); + sect->fileoffset = FileIO::ftell64(m_ReadFileHandle); + + FileIO::fread(§->data[0], 1, (size_t)length, m_ReadFileHandle); + + if(sect->type != eSectionType_Unknown && sect->type < eSectionType_Num) + m_KnownSections[sect->type] = sect; + m_Sections.push_back(sect); + } + else if(sectionHeader.isASCII == 0x0) + { + FileIO::fread(reading, 1, offsetof(BinarySectionHeader, name)-1, m_ReadFileHandle); + + Section *sect = new Section(); + sect->flags = sectionHeader.sectionFlags; + sect->type = sectionHeader.sectionType; + sect->name.resize(sectionHeader.sectionNameLength-1); + sect->size = sectionHeader.sectionLength; + + FileIO::fread(§->name[0], 1, sectionHeader.sectionNameLength-1, m_ReadFileHandle); + char nullterm = 0; + FileIO::fread(&nullterm, 1, 1, m_ReadFileHandle); + + sect->fileoffset = FileIO::ftell64(m_ReadFileHandle); + + if(sect->flags & eSectionFlag_LZ4Compressed) + { + sect->compressedReader = new CompressedFileIO(m_ReadFileHandle); + FileIO::fread(§->size, 1, sizeof(uint64_t), m_ReadFileHandle); + + sect->fileoffset += sizeof(uint64_t); + } + + if(sect->type != eSectionType_Unknown && sect->type < eSectionType_Num) + m_KnownSections[sect->type] = sect; + m_Sections.push_back(sect); + + // if section isn't frame capture data and is small enough, read it all into memory now, otherwise skip + if(sect->type != eSectionType_FrameCapture && sectionHeader.sectionLength < 4*1024*1024) + { + sect->data.resize(sectionHeader.sectionLength); + FileIO::fread(§->data[0], 1, sectionHeader.sectionLength, m_ReadFileHandle); + } + else + { + FileIO::fseek64(m_ReadFileHandle, sectionHeader.sectionLength, SEEK_CUR); + } + } + else + { + RETURNCORRUPT("Unrecognised section type '%hhx'", sectionHeader.isASCII); + } + } + } + else { RDCERR("Capture file from wrong version. This program is logfile version %llu, file is logfile version %llu", SERIALISE_VERSION, header.version); @@ -284,13 +832,10 @@ Serialiser::Serialiser(const char *path, Mode mode, bool debugMode) m_ReadFileHandle = 0; return; } - - FileIO::fseek64(m_ReadFileHandle, 0, SEEK_END); - - uint64_t realLength = FileIO::ftell64(m_ReadFileHandle); - if(header.fileSize != realLength) + + if(m_KnownSections[eSectionType_FrameCapture] == NULL) { - RDCERR("Truncated/overlong capture file. Expected length 0x016llx, got 0x016llx", header.fileSize, realLength); + RDCERR("Capture file doesn't have a frame capture"); m_ErrorCode = eSerError_Corrupt; m_HasError = true; @@ -299,20 +844,19 @@ Serialiser::Serialiser(const char *path, Mode mode, bool debugMode) return; } - m_HasResolver = header.resolveDBSize > 0; - - m_FileStartOffset = AlignUp16(sizeof(DebuggerHeader) + header.resolveDBSize); - - m_BufferSize = realLength-m_FileStartOffset; + m_BufferSize = m_KnownSections[eSectionType_FrameCapture]->size; m_CurrentBufferSize = (size_t)RDCMIN(m_BufferSize, (uint64_t)64*1024); m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); m_ReadOffset = 0; + FileIO::fseek64(m_ReadFileHandle, m_KnownSections[eSectionType_FrameCapture]->fileoffset, SEEK_SET); + + // read initial buffer of data ReadFromFile(0, m_CurrentBufferSize); } else { - m_pResolver = NULL; + m_SerVer = SERIALISE_VERSION; if(m_Filename != "") { @@ -324,21 +868,209 @@ Serialiser::Serialiser(const char *path, Mode mode, bool debugMode) m_BufferSize = 128*1024; m_BufferHead = m_Buffer = AllocAlignedBuffer((size_t)m_BufferSize); } - - m_ReadOffset = 0; - m_FileStartOffset = 0; } } -void Serialiser::ReadFromFile(uint64_t destOffs, size_t chunkLen) +void Serialiser::Reset() +{ + if(m_ResolverThread != 0) + { + m_ResolverThreadKillSignal = true; + + Threading::JoinThread(m_ResolverThread); + Threading::CloseThread(m_ResolverThread); + m_ResolverThread = 0; + } + + m_DebugText = ""; + m_DebugTextWriting = false; + + RDCEraseEl(m_KnownSections); + + m_HasError = false; + m_ErrorCode = eSerError_None; + + m_Mode = NONE; + + m_Indent = 0; + + SAFE_DELETE(m_pCallstack); + SAFE_DELETE(m_pResolver); + if(m_Buffer) + { + FreeAlignedBuffer(m_Buffer); + m_Buffer = NULL; + } + + m_ChunkLookup = NULL; + + m_AlignedData = false; + + m_ReadFileHandle = NULL; + + m_ReadOffset = 0; + + m_BufferHead = m_Buffer = NULL; + m_CurrentBufferSize = 0; + m_BufferSize = 0; +} + +Serialiser::~Serialiser() +{ + if(m_ResolverThread != 0) + { + m_ResolverThreadKillSignal = true; + Threading::JoinThread(m_ResolverThread); + Threading::CloseThread(m_ResolverThread); + m_ResolverThread = 0; + } + + if(m_ReadFileHandle) + { + FileIO::fclose(m_ReadFileHandle); + m_ReadFileHandle = 0; + } + + for(size_t i=0; i < m_Sections.size(); i++) + SAFE_DELETE(m_Sections[i]->compressedReader); + + for(size_t i=0; i < m_Chunks.size(); i++) + { + if(m_Chunks[i]->IsTemporary()) + SAFE_DELETE(m_Chunks[i]); + } + + m_Chunks.clear(); + + SAFE_DELETE(m_pResolver); + SAFE_DELETE(m_pCallstack); + if(m_Buffer) + { + FreeAlignedBuffer(m_Buffer); + m_Buffer = NULL; + } + m_Buffer = NULL; + m_BufferHead = NULL; +} + +void Serialiser::WriteBytes( const byte *buf, size_t nBytes ) +{ +#ifdef DEBUG_TEXT_SERIALISER + if(m_Mode == DEBUGWRITING) + return; +#endif + + if(m_HasError) + { + RDCERR("Writing bytes with error state serialiser"); + return; + } + + if(m_Buffer+m_BufferSize < m_BufferHead+nBytes+8) + { + // reallocate + while(m_Buffer+m_BufferSize < m_BufferHead+nBytes+8) + { + m_BufferSize += 128*1024; + } + + byte *newBuf = AllocAlignedBuffer((size_t)m_BufferSize); + + size_t curUsed = m_BufferHead-m_Buffer; + + memcpy(newBuf, m_Buffer, curUsed); + + FreeAlignedBuffer(m_Buffer); + + m_Buffer = newBuf; + m_BufferHead = newBuf + curUsed; + } + + memcpy(m_BufferHead, buf, nBytes); + + m_BufferHead += nBytes; +} + +void * Serialiser::ReadBytes( size_t nBytes ) +{ + if(m_HasError) + { + RDCERR("Reading bytes with error state serialiser"); + return NULL; + } + + // if we would read off the end of our current window + if(m_BufferHead+nBytes > m_Buffer+m_CurrentBufferSize) + { + // store old buffer and the read data, so we can move it into the new buffer + byte *oldBuffer = m_Buffer; + + // always keep at least a certain window behind what we read. + size_t backwardsWindow = RDCMIN((size_t)64, size_t(m_BufferHead - m_Buffer)); + + byte *currentData = m_BufferHead - backwardsWindow; + size_t currentDataSize = m_CurrentBufferSize - (m_BufferHead-m_Buffer) + backwardsWindow; + + size_t BufferOffset = m_BufferHead-m_Buffer; + + // if we are reading more than our current buffer size, expand the buffer size + if(nBytes+backwardsWindow > m_CurrentBufferSize) + { + // very conservative resizing - don't do "double and add" - to avoid + // a 1GB buffer being read and needing to allocate 2GB. The cost is we + // will reallocate a bit more often + m_CurrentBufferSize = nBytes+backwardsWindow; + m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); + } + + // move the unread data into the buffer + memmove(m_Buffer, currentData, currentDataSize); + + if(BufferOffset > backwardsWindow) + { + m_ReadOffset += BufferOffset-backwardsWindow; + m_BufferHead = m_Buffer+backwardsWindow; + } + else + { + m_BufferHead = m_Buffer+BufferOffset; + } + + // if there's anything left of the file to read in, do so now + ReadFromFile(currentDataSize, RDCMIN(m_CurrentBufferSize-currentDataSize, size_t(m_BufferSize - m_ReadOffset - currentDataSize))); + + if(oldBuffer != m_Buffer) + FreeAlignedBuffer(oldBuffer); + } + + void *ret = m_BufferHead; + + m_BufferHead += nBytes; + + RDCASSERT(m_BufferHead <= m_Buffer+m_CurrentBufferSize); + + return ret; +} + +void Serialiser::ReadFromFile(uint64_t bufferOffs, size_t length) { RDCASSERT(m_ReadFileHandle); if(m_ReadFileHandle == NULL) return; - FileIO::fseek64(m_ReadFileHandle, m_FileStartOffset+destOffs, SEEK_SET); - FileIO::fread(m_Buffer + destOffs - m_ReadOffset, 1, chunkLen, m_ReadFileHandle); + Section *s = m_KnownSections[eSectionType_FrameCapture]; + + RDCASSERT(s); + + if(s->flags & eSectionFlag_LZ4Compressed) + { + s->compressedReader->Read(m_Buffer + bufferOffs, length); + } + else + { + FileIO::fread(m_Buffer + bufferOffs, 1, length, m_ReadFileHandle); + } } byte *Serialiser::AllocAlignedBuffer(size_t size, size_t alignment) @@ -381,9 +1113,80 @@ void Serialiser::FreeAlignedBuffer(byte *buf) delete[] rawAlloc; } +void Serialiser::SetPersistentBlock(uint64_t offs) +{ + FreeAlignedBuffer(m_Buffer); + + RDCASSERT(m_BufferSize - offs < 0xffffffff); + + m_CurrentBufferSize = (size_t)(m_BufferSize - offs); + m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); + m_ReadOffset = offs; + + RDCASSERT(m_ReadFileHandle); + + Section *s = m_KnownSections[eSectionType_FrameCapture]; + RDCASSERT(s); + FileIO::fseek64(m_ReadFileHandle, s->fileoffset, SEEK_SET); + + s->compressedReader->Reset(); + + // can't seek arbitrarily in the stream, need to read through the rest + while(offs > 0) + { + // read at most buffer size at a time. + size_t chunkSize = RDCMIN(m_CurrentBufferSize, (size_t)offs); + ReadFromFile(0, chunkSize); + offs -= chunkSize; + } + + // now read the rest + ReadFromFile(0, m_CurrentBufferSize); + FileIO::fclose(m_ReadFileHandle); + m_ReadFileHandle = 0; +} + +void Serialiser::SetOffset(uint64_t offs) +{ + if(m_HasError) + { + RDCERR("Setting offset with error state serialiser"); + return; + } + + // if we're jumping back before our in-memory window just reset the window + // and load it all in from scratch. + if(m_Mode == READING && offs < m_ReadOffset) + { + // if we're reading from file, only support rewinding all the way to the start + RDCASSERT(m_ReadFileHandle == NULL || offs == 0); + + if(m_ReadFileHandle) + { + Section *s = m_KnownSections[eSectionType_FrameCapture]; + RDCASSERT(s); + FileIO::fseek64(m_ReadFileHandle, s->fileoffset, SEEK_SET); + + s->compressedReader->Reset(); + } + + FreeAlignedBuffer(m_Buffer); + + m_CurrentBufferSize = (size_t)RDCMIN(m_BufferSize, (uint64_t)64*1024); + m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); + m_ReadOffset = offs; + + ReadFromFile(0, m_CurrentBufferSize); + } + + RDCASSERT(m_BufferHead && m_Buffer && offs <= GetSize()); + m_BufferHead = m_Buffer + offs - m_ReadOffset; + m_Indent = 0; +} + void Serialiser::InitCallstackResolver() { - if(m_pResolver == NULL && m_ResolverThread == 0) + if(m_pResolver == NULL && m_ResolverThread == 0 && m_KnownSections[eSectionType_ResolveDatabase] != NULL) { m_ResolverThreadKillSignal = false; m_ResolverThread = Threading::CreateThread(&Serialiser::CreateResolver, (void *)this); @@ -401,69 +1204,19 @@ void Serialiser::SetCallstack(uint64_t *levels, size_t numLevels) void Serialiser::CreateResolver(void *ths) { Serialiser *ser = (Serialiser *)ths; - FILE *binFile = FileIO::fopen(ser->m_Filename.c_str(), "rb"); - if(!binFile) - { - RDCERR("Can't open capture file '%s' for read - errno %d", ser->m_Filename.c_str(), errno); - return; - } - - DebuggerHeader header; - FileIO::fread(&header, 1, sizeof(DebuggerHeader), binFile); - - if(header.magic != MAGIC_HEADER) - { - RDCERR("Invalid capture file. Expected 0x%08lx, got 0x%08llx", MAGIC_HEADER, header.magic); - FileIO::fclose(binFile); - return; - } - - if(header.version != SERIALISE_VERSION) - { - RDCERR("Capture file from wrong version. This program is logfile version %llu, file is logfile version %llu", SERIALISE_VERSION, header.version); - FileIO::fclose(binFile); - return; - } - - FileIO::fseek64(binFile, 0, SEEK_END); - - uint64_t realLength = FileIO::ftell64(binFile); - if(header.fileSize != realLength) - { - RDCERR("Truncated/overlong capture file. Expected length 0x08llx, got 0x08lx", header.fileSize, realLength); - FileIO::fclose(binFile); - return; - } - - if(header.resolveDBSize == 0) - { - RDCWARN("Trying to create resolver when no resolve DB is in the capture file"); - FileIO::fclose(binFile); - return; - } - - FileIO::fseek64(binFile, sizeof(DebuggerHeader), SEEK_SET); - - RDCASSERT(header.resolveDBSize < 0xffffffff); - - char *resolveDB = new char[(size_t)header.resolveDBSize]; - - FileIO::fread(resolveDB, 1, (size_t)header.resolveDBSize, binFile); - - FileIO::fclose(binFile); - string dir = dirname(ser->m_Filename); - Callstack::StackResolver *resolver = Callstack::MakeResolver(resolveDB, (size_t)header.resolveDBSize, dir, &ser->m_ResolverThreadKillSignal); + Section *s = ser->m_KnownSections[Serialiser::eSectionType_ResolveDatabase]; + RDCASSERT(s); - ser->m_pResolver = resolver; - - SAFE_DELETE_ARRAY(resolveDB); + ser->m_pResolver = Callstack::MakeResolver((char *)&s->data[0], s->data.size(), dir, &ser->m_ResolverThreadKillSignal); } -uint64_t Serialiser::FlushToDisk() +void Serialiser::FlushToDisk() { + SCOPED_TIMER("File writing"); + if(m_Filename != "" && !m_HasError && m_Mode == WRITING) { RDCDEBUG("writing capture files"); @@ -484,20 +1237,124 @@ uint64_t Serialiser::FlushToDisk() } } - FILE *binFile = FileIO::fopen(m_Filename.c_str(), "wb"); + FILE *binFile = FileIO::fopen(m_Filename.c_str(), "w+b"); if(!binFile) { RDCERR("Can't open capture file '%s' for write, errno %d", m_Filename.c_str(), errno); m_ErrorCode = eSerError_FileIO; m_HasError = true; - return 0; + return; } RDCDEBUG("Opened capture file for write"); - DebuggerHeader header; + FileHeader header; // automagically initialised with correct data + + // write header + FileIO::fwrite(&header, 1, sizeof(FileHeader), binFile); + + static const byte padding[BufferAlignment] = {0}; + uint64_t compressedSizeOffset = 0; + uint64_t uncompressedSizeOffset = 0; + + // write frame capture section header + { + const char sectionName[] = "renderdoc/internal/framecapture"; + + BinarySectionHeader section = { 0 }; + section.isASCII = 0; // redundant but explicit + section.sectionNameLength = sizeof(sectionName); // includes null terminator + section.sectionType = eSectionType_FrameCapture; + section.sectionFlags = eSectionFlag_LZ4Compressed; + section.sectionLength = 0; // will be fixed up later, to avoid having to compress everything into memory + + compressedSizeOffset = FileIO::ftell64(binFile) + offsetof(BinarySectionHeader, sectionLength); + + FileIO::fwrite(§ion, 1, offsetof(BinarySectionHeader, name), binFile); + FileIO::fwrite(sectionName, 1, sizeof(sectionName), binFile); + + uint64_t len = 0; // will be fixed up later + uncompressedSizeOffset = FileIO::ftell64(binFile); + FileIO::fwrite(&len, 1, sizeof(uint64_t), binFile); + } + + CompressedFileIO fwriter(binFile); + + // track offset so we can add padding. The padding is relative + // to the start of the decompressed buffer, so we start it from 0 + uint64_t offs = 0; + uint64_t alignedoffs = 0; + + // write frame capture contents + for(size_t i=0; i < m_Chunks.size(); i++) + { + Chunk *chunk = m_Chunks[i]; + + alignedoffs = AlignUp(offs, BufferAlignment); + + if(offs != alignedoffs && chunk->IsAligned()) + { + uint16_t chunkIdx = 0; // write a '0' chunk that indicates special behaviour + fwriter.Write(&chunkIdx, sizeof(chunkIdx)); + offs += sizeof(chunkIdx); + + uint8_t controlByte = 0; // control byte 0 indicates padding + fwriter.Write(&controlByte, sizeof(controlByte)); + offs += sizeof(controlByte); + + offs++; // we will have to write out a byte indicating how much padding exists, so add 1 + alignedoffs = AlignUp(offs, BufferAlignment); + + RDCCOMPILE_ASSERT(BufferAlignment < 0x100, "Buffer alignment must be less than 256"); // with a byte at most indicating how many bytes to pad, + // this is our maximal representable alignment + + uint8_t padLength = (alignedoffs-offs)&0xff; + fwriter.Write(&padLength, sizeof(padLength)); + + // we might have padded with the control bytes, so only write some bytes if we need to + if(padLength > 0) + { + fwriter.Write(padding, size_t(alignedoffs-offs)); + offs += alignedoffs-offs; + } + } + + fwriter.Write(chunk->GetData(), chunk->GetLength()); + + offs += chunk->GetLength(); + + if(chunk->IsTemporary()) + SAFE_DELETE(chunk); + } + + fwriter.Flush(); + + m_Chunks.clear(); + + // fixup section size + { + uint32_t compsize = 0; + uint64_t uncompsize = 0; + + uint64_t curoffs = FileIO::ftell64(binFile); + + FileIO::fseek64(binFile, compressedSizeOffset, SEEK_SET); + + compsize = fwriter.GetCompressedSize(); + FileIO::fwrite(&compsize, 1, sizeof(compsize), binFile); + + FileIO::fseek64(binFile, uncompressedSizeOffset, SEEK_SET); + + uncompsize = fwriter.GetUncompressedSize(); + FileIO::fwrite(&uncompsize, 1, sizeof(uncompsize), binFile); + + FileIO::fseek64(binFile, curoffs, SEEK_SET); + + RDCLOG("Compressed frame capture data from %u to %u", fwriter.GetUncompressedSize(), fwriter.GetCompressedSize()); + } + char *symbolDB = NULL; size_t symbolDBSize = 0; @@ -513,118 +1370,28 @@ uint64_t Serialiser::FlushToDisk() Callstack::GetLoadedModules(symbolDB, symbolDBSize); } - header.resolveDBSize = symbolDBSize; - - // write header - FileIO::fwrite(&header, 1, sizeof(DebuggerHeader), binFile); - - // write symbol database - if(symbolDBSize > 0) - FileIO::fwrite(symbolDB, 1, symbolDBSize, binFile); - - static const byte padding[BufferAlignment] = {0}; - - size_t offs = sizeof(DebuggerHeader) + symbolDBSize; - size_t alignedoffs = AlignUp16(offs); - - FileIO::fwrite(padding, 1, alignedoffs-offs, binFile); - - offs = alignedoffs; - - // write serialise contents - for(size_t i=0; i < m_Chunks.size(); i++) + // write symbol database section + if(symbolDB) { - Chunk *chunk = m_Chunks[i]; + const char sectionName[] = "renderdoc/internal/resolvedb"; - alignedoffs = AlignUp16(offs); - - if(offs != alignedoffs && chunk->IsAligned()) - { - uint16_t chunkIdx = 0; // write a '0' chunk that indicates special behaviour - FileIO::fwrite(&chunkIdx, sizeof(chunkIdx), 1, binFile); - offs += sizeof(chunkIdx); - - uint8_t controlByte = 0; // control byte 0 indicates padding - FileIO::fwrite(&controlByte, sizeof(controlByte), 1, binFile); - offs += sizeof(controlByte); - - offs++; // we will have to write out a byte indicating how much padding exists, so add 1 - alignedoffs = AlignUp16(offs); - - RDCCOMPILE_ASSERT(BufferAlignment < 0x100, "Buffer alignment must be less than 256"); // with a byte at most indicating how many bytes to pad, - // this is our maximal representable alignment - - uint8_t padLength = (alignedoffs-offs)&0xff; - FileIO::fwrite(&padLength, sizeof(padLength), 1, binFile); - - // we might have padded with the control bytes, so only write some bytes if we need to - if(padLength > 0) - { - FileIO::fwrite(padding, 1, alignedoffs-offs, binFile); - offs += alignedoffs-offs; - } - } + BinarySectionHeader section = { 0 }; + section.isASCII = 0; // redundant but explicit + section.sectionNameLength = sizeof(sectionName); // includes null terminator + section.sectionType = eSectionType_ResolveDatabase; + section.sectionLength = (uint32_t)symbolDBSize; - FileIO::fwrite(chunk->GetData(), 1, chunk->GetLength(), binFile); + FileIO::fwrite(§ion, 1, offsetof(BinarySectionHeader, name), binFile); + FileIO::fwrite(sectionName, 1, sizeof(sectionName), binFile); - offs += chunk->GetLength(); + // write actual data + FileIO::fwrite(symbolDB, 1, symbolDBSize, binFile); - if(chunk->IsTemporary()) - SAFE_DELETE(chunk); + SAFE_DELETE_ARRAY(symbolDB); } - m_Chunks.clear(); - - header.fileSize = offs; - - FileIO::fseek64(binFile, 0, SEEK_SET); - - // write header with correct filesize (accounting for all padding) - FileIO::fwrite(&header, 1, sizeof(DebuggerHeader), binFile); - FileIO::fclose(binFile); - - SAFE_DELETE_ARRAY(symbolDB); - - return header.fileSize; } - - return 0; -} - -Serialiser::~Serialiser() -{ - if(m_ResolverThread != 0) - { - m_ResolverThreadKillSignal = true; - Threading::JoinThread(m_ResolverThread); - Threading::CloseThread(m_ResolverThread); - m_ResolverThread = 0; - } - - if(m_ReadFileHandle) - { - FileIO::fclose(m_ReadFileHandle); - m_ReadFileHandle = 0; - } - - for(size_t i=0; i < m_Chunks.size(); i++) - { - if(m_Chunks[i]->IsTemporary()) - SAFE_DELETE(m_Chunks[i]); - } - - m_Chunks.clear(); - - SAFE_DELETE(m_pResolver); - SAFE_DELETE(m_pCallstack); - if(m_Buffer) - { - FreeAlignedBuffer(m_Buffer); - m_Buffer = NULL; - } - m_Buffer = NULL; - m_BufferHead = NULL; } void Serialiser::DebugPrint(const char *fmt, ...) @@ -921,27 +1688,12 @@ void Serialiser::Insert(Chunk *chunk) m_DebugText += chunk->GetDebugString(); } -void Serialiser::SkipBuffer() -{ - RDCASSERT(m_Mode < WRITING); - - uint32_t len; - ReadInto(len); - - // ensure byte alignment - uint64_t offs = GetOffset(); - uint64_t alignedoffs = AlignUp16(offs); - - if(offs != alignedoffs) - { - ReadBytes((size_t)(alignedoffs-offs)); - } - - ReadBytes(len); -} - void Serialiser::AlignNextBuffer(const size_t alignment) { + // on new logs, we don't have to align. This code will be deleted once backwards-compat is dropped + if(m_Mode >= WRITING || m_SerVer >= 0x00000032) + return; + // this is a super hack but it's the easiest way to align a buffer to a larger pow2 alignment // than the default 16-bytes, while still able to be backwards compatible with old logs that // weren't so aligned. We know that SerialiseBuffer will align to the nearest 16-byte boundary @@ -991,7 +1743,7 @@ void Serialiser::SerialiseBuffer(const char *name, byte *&buf, size_t &len) // ensure byte alignment uint64_t offs = GetOffset(); - uint64_t alignedoffs = AlignUp16(offs); + uint64_t alignedoffs = AlignUp(offs, BufferAlignment); if(offs != alignedoffs) { @@ -1011,7 +1763,9 @@ void Serialiser::SerialiseBuffer(const char *name, byte *&buf, size_t &len) // ensure byte alignment uint64_t offs = GetOffset(); - uint64_t alignedoffs = AlignUp16(offs); + + // serialise version 0x00000031 had only 16-byte alignment + uint64_t alignedoffs = AlignUp(offs, m_SerVer == 0x00000031 ? 16 : BufferAlignment); if(offs != alignedoffs) { diff --git a/renderdoc/serialise/serialiser.h b/renderdoc/serialise/serialiser.h index ff8b6310c..7525f5c65 100644 --- a/renderdoc/serialise/serialiser.h +++ b/renderdoc/serialise/serialiser.h @@ -80,6 +80,7 @@ typedef const char *(*ChunkLookup)(uint32_t chunkType); class Serialiser; class ScopedContext; +struct CompressedFileIO; // holds the memory, length and type for a given chunk, so that it can be // passed around and moved between owners before being serialised out @@ -105,7 +106,7 @@ class Chunk #endif // grab current contents of the serialiser into this chunk - Chunk(Serialiser *ser, uint32_t chunkType, size_t alignment, bool temp); + Chunk(Serialiser *ser, uint32_t chunkType, bool temp); private: // no copy semantics @@ -162,10 +163,28 @@ class Serialiser eSerError_UnsupportedVersion, }; + enum SectionFlags + { + eSectionFlag_None = 0x0, + eSectionFlag_ASCIIStored = 0x1, + eSectionFlag_LZ4Compressed = 0x2, + }; + + enum SectionType + { + eSectionType_Unknown = 0, + eSectionType_FrameCapture, // renderdoc/internal/framecapture + eSectionType_ResolveDatabase, // renderdoc/internal/resolvedb + eSectionType_FrameBookmarks, // renderdoc/ui/bookmarks + eSectionType_Notes, // renderdoc/ui/notes + eSectionType_Num, + }; + // version number of overall file format or chunk organisation. If the contents/meaning/order of // chunks have changed this does not need to be bumped, there are version numbers within each // API that interprets the stream that can be bumped. - static const uint64_t SERIALISE_VERSION = 0x00000031; + static const uint64_t SERIALISE_VERSION = 0x00000032; + static const uint32_t MAGIC_HEADER; ////////////////////////////////////////// // Init and error handling @@ -228,46 +247,9 @@ class Serialiser // Set up the base pointer and size. Serialiser will allocate enough for // the rest of the file and keep it all in memory (useful to keep everything // in actual frame data resident in memory). - void SetBase(uint64_t offs) - { - FreeAlignedBuffer(m_Buffer); + void SetPersistentBlock(uint64_t offs); - RDCASSERT(m_BufferSize - offs < 0xffffffff); - - m_CurrentBufferSize = (size_t)(m_BufferSize - offs); - m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); - m_ReadOffset = offs; - - ReadFromFile(offs, m_CurrentBufferSize); - FileIO::fclose(m_ReadFileHandle); - m_ReadFileHandle = 0; - } - - void SetOffset(uint64_t offs) - { - if(m_HasError) - { - RDCERR("Setting offset with error state serialiser"); - return; - } - - // if we're jumping back before our in-memory window just reset the window - // and load it all in from scratch. - if(m_Mode == READING && offs < m_ReadOffset) - { - FreeAlignedBuffer(m_Buffer); - - m_CurrentBufferSize = (size_t)RDCMIN(m_BufferSize, (uint64_t)64*1024); - m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); - m_ReadOffset = offs; - - ReadFromFile(offs, m_CurrentBufferSize); - } - - RDCASSERT(m_BufferHead && m_Buffer && offs <= GetSize()); - m_BufferHead = m_Buffer + offs - m_ReadOffset; - m_Indent = 0; - } + void SetOffset(uint64_t offs); void Rewind() { @@ -311,7 +293,7 @@ class Serialiser } void InitCallstackResolver(); - bool HasCallstacks() { return m_HasResolver; } + bool HasCallstacks() { return m_KnownSections[eSectionType_ResolveDatabase] != NULL; } // get callstack resolver, created with the DB in the file Callstack::StackResolver *GetCallstackResolver() @@ -553,7 +535,6 @@ class Serialiser // If serialising in, buf must either be NULL in which case allocated // memory will be returned, or it must be already large enough. void SerialiseBuffer(const char *name, byte *&buf, size_t &len); - void SkipBuffer(); void AlignNextBuffer(const size_t alignment); // NOT recommended interface. Useful for specific situations if e.g. you have @@ -573,10 +554,10 @@ class Serialiser // prints to the debug output log void DebugPrint(const char *fmt, ...); - static byte *AllocAlignedBuffer(size_t size, size_t align = 16); + static byte *AllocAlignedBuffer(size_t size, size_t align = 64); static void FreeAlignedBuffer(byte *buf); - uint64_t FlushToDisk(); + void FlushToDisk(); // set a function used when serialising a text representation // of the chunks @@ -616,87 +597,10 @@ class Serialiser ////////////////////////////////////////// // Raw memory buffer read/write - void WriteBytes(const byte *buf, size_t nBytes) - { -#ifdef DEBUG_TEXT_SERIALISER - if(m_Mode == DEBUGWRITING) - return; -#endif + void WriteBytes(const byte *buf, size_t nBytes); + void *ReadBytes(size_t nBytes); - if(m_HasError) - { - RDCERR("Writing bytes with error state serialiser"); - return; - } - - if(m_Buffer+m_BufferSize < m_BufferHead+nBytes+8) - { - // reallocate - while(m_Buffer+m_BufferSize < m_BufferHead+nBytes+8) - { - m_BufferSize += 128*1024; - } - - byte *newBuf = AllocAlignedBuffer((size_t)m_BufferSize); - - size_t curUsed = m_BufferHead-m_Buffer; - - memcpy(newBuf, m_Buffer, curUsed); - - FreeAlignedBuffer(m_Buffer); - - m_Buffer = newBuf; - m_BufferHead = newBuf + curUsed; - } - - memcpy(m_BufferHead, buf, nBytes); - - m_BufferHead += nBytes; - } - void *ReadBytes(size_t nBytes) - { - if(m_HasError) - { - RDCERR("Reading bytes with error state serialiser"); - return NULL; - } - - // if we would read off the end of our current window - if(m_BufferHead+nBytes > m_Buffer+m_CurrentBufferSize) - { - size_t BufferOffset = m_BufferHead-m_Buffer; - - if(nBytes+64 > m_CurrentBufferSize) - { - FreeAlignedBuffer(m_Buffer); - m_CurrentBufferSize = nBytes+64; - m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); - } - - if(BufferOffset > 64) - { - m_ReadOffset += BufferOffset-64; - m_BufferHead = m_Buffer+64; - } - else - { - m_BufferHead = m_Buffer+BufferOffset; - } - - // if there's anything left of the file to read in, do so now - ReadFromFile(m_ReadOffset, RDCMIN(m_CurrentBufferSize, (size_t)(m_BufferSize-m_ReadOffset))); - } - - void *ret = m_BufferHead; - - m_BufferHead += nBytes; - - RDCASSERT(m_BufferHead <= m_Buffer+m_CurrentBufferSize); - - return ret; - } - - void ReadFromFile(uint64_t destOffs, size_t chunkLen); + void ReadFromFile(uint64_t bufferOffs, size_t length); template void WriteFrom(const T &f) { @@ -733,25 +637,11 @@ class Serialiser ////////////////////////////////////////// - static const uint32_t MAGIC_HEADER; - - static const size_t BufferAlignment; - - struct DebuggerHeader - { - DebuggerHeader() - { - magic = MAGIC_HEADER; - version = SERIALISE_VERSION; - } - - uint64_t magic; - uint64_t version; - uint64_t fileSize; - uint64_t resolveDBSize; - }; + static const uint64_t BufferAlignment; ////////////////////////////////////////// + + uint64_t m_SerVer; Mode m_Mode; @@ -761,7 +651,6 @@ class Serialiser int m_Indent; - bool m_HasResolver; Callstack::Stackwalk *m_pCallstack; Callstack::StackResolver *m_pResolver; Threading::ThreadHandle m_ResolverThread; @@ -779,11 +668,27 @@ class Serialiser // reading from file: - // where in the actual on-disk file does the data start (ie. after header and symbol DB) - uint64_t m_FileStartOffset; + struct Section + { + Section() : type(eSectionType_Unknown), flags(eSectionFlag_None), fileoffset(0), compressedReader(NULL) {} + string name; + SectionType type; + SectionFlags flags; + + uint64_t fileoffset; + uint64_t size; + vector data; // some sections can be loaded entirely into memory + CompressedFileIO *compressedReader; + }; + + // this lists all sections in file order + vector m_Sections; + + // this lists known sections, some may be NULL + Section *m_KnownSections[eSectionType_Num]; // where does our in-memory window point to in the data stream. ie. m_pBuffer[0] is - // m_ReadOffset into the disk stream + // m_ReadOffset into the frame capture section uint64_t m_ReadOffset; // how big is the current in-memory window @@ -821,7 +726,6 @@ class ScopedContext , m_DebugSer(debugser) #endif { - m_Alignment = 0; m_Name = string(n) + " = " + t; m_Ser->PushContext(m_Name.c_str(), m_Idx, smallChunk); @@ -839,7 +743,6 @@ class ScopedContext , m_DebugSer(debugser) #endif { - m_Alignment = 0; m_Name = n; m_Ser->PushContext(m_Name.c_str(), m_Idx, smallChunk); @@ -857,20 +760,14 @@ class ScopedContext End(); } - void SetAlignment(size_t align) - { - m_Alignment = align; - } - Chunk *Get(bool temporary = false) { End(); - return new Chunk(m_Ser, m_Idx, m_Alignment, temporary); + return new Chunk(m_Ser, m_Idx, temporary); } private: std::string m_Name; uint32_t m_Idx; - size_t m_Alignment; Serialiser *m_Ser; #ifdef DEBUG_TEXT_SERIALISER Serialiser *m_DebugSer;