diff --git a/renderdoc/common/common.h b/renderdoc/common/common.h index 1c10a7edc..e36cbb8b3 100644 --- a/renderdoc/common/common.h +++ b/renderdoc/common/common.h @@ -85,6 +85,9 @@ inline T AlignUp16(T x) { return (x+0xf) & (~0xf); } template inline T AlignUp(T x, T a) { return (x+(a-1)) & (~(a-1)); } +template +inline T AlignUpPtr(T x, A a) { return (T)AlignUp( (uintptr_t)x, (uintptr_t)a ); } + #define MAKE_FOURCC(a, b, c, d) (((uint32_t)(d) << 24) | ((uint32_t)(c) << 16) | ((uint32_t)(b) << 8) | (uint32_t)(a)) bool FindDiffRange(void *a, void *b, size_t bufSize, size_t &diffStart, size_t &diffEnd); diff --git a/renderdoc/driver/d3d11/d3d11_context_wrap.cpp b/renderdoc/driver/d3d11/d3d11_context_wrap.cpp index 0c41c188a..86018cb0a 100644 --- a/renderdoc/driver/d3d11/d3d11_context_wrap.cpp +++ b/renderdoc/driver/d3d11/d3d11_context_wrap.cpp @@ -5182,8 +5182,6 @@ void WrappedID3D11DeviceContext::UpdateSubresource(ID3D11Resource *pDstResource, Serialise_UpdateSubresource(pDstResource, DstSubresource, pDstBox, pSrcData, SrcRowPitch, SrcDepthPitch); - scope.SetAlignment(32); - Chunk *chunk = scope.Get(); record->AddChunk(chunk); @@ -7133,8 +7131,6 @@ void WrappedID3D11DeviceContext::Unmap(ID3D11Resource *pResource, UINT Subresour m_pSerialiser->Serialise("context", m_ResourceID); Serialise_Unmap(pResource, Subresource); - scope.SetAlignment(32); - m_ContextRecord->AddChunk(scope.Get()); } else if(m_State >= WRITING) @@ -7162,8 +7158,6 @@ void WrappedID3D11DeviceContext::Unmap(ID3D11Resource *pResource, UINT Subresour m_pSerialiser->Serialise("context", m_ResourceID); Serialise_Unmap(pResource, Subresource); - scope.SetAlignment(32); - Chunk *chunk = scope.Get(); record->AddChunk(chunk); diff --git a/renderdoc/driver/d3d11/d3d11_device.cpp b/renderdoc/driver/d3d11/d3d11_device.cpp index f9ca6b0e4..5c438d941 100644 --- a/renderdoc/driver/d3d11/d3d11_device.cpp +++ b/renderdoc/driver/d3d11/d3d11_device.cpp @@ -295,8 +295,6 @@ WrappedID3D11Device::WrappedID3D11Device(ID3D11Device* realDevice, D3D11InitPara m_AppControlledCapture = false; m_TotalTime = m_AvgFrametime = m_MinFrametime = m_MaxFrametime = 0.0; - - m_CurFileSize = 0; #if defined(RELEASE) const bool debugSerialiser = false; @@ -1106,7 +1104,7 @@ void WrappedID3D11Device::ReadLogInitialisation() m_pSerialiser->SetDebugText(false); - m_pSerialiser->SetBase(firstFrame); + m_pSerialiser->SetPersistentBlock(firstFrame); } bool WrappedID3D11Device::Prepare_InitialState(ID3D11DeviceChild *res) @@ -2834,7 +2832,7 @@ bool WrappedID3D11Device::EndFrameCapture(void *dev, void *wnd) RDCDEBUG("Done"); } - m_CurFileSize += m_pFileSerialiser->FlushToDisk(); + m_pFileSerialiser->FlushToDisk(); SAFE_DELETE(m_pFileSerialiser); diff --git a/renderdoc/driver/d3d11/d3d11_device.h b/renderdoc/driver/d3d11/d3d11_device.h index 23c656a8a..0c76c7d0e 100644 --- a/renderdoc/driver/d3d11/d3d11_device.h +++ b/renderdoc/driver/d3d11/d3d11_device.h @@ -261,8 +261,6 @@ private: CaptureFailReason m_FailedReason; uint32_t m_Failures; - uint64_t m_CurFileSize; - PerformanceTimer m_FrameTimer; vector m_FrameTimes; double m_TotalTime, m_AvgFrametime, m_MinFrametime, m_MaxFrametime; diff --git a/renderdoc/driver/d3d11/d3d11_device_wrap.cpp b/renderdoc/driver/d3d11/d3d11_device_wrap.cpp index c0f32ae7c..7eaa83fbd 100644 --- a/renderdoc/driver/d3d11/d3d11_device_wrap.cpp +++ b/renderdoc/driver/d3d11/d3d11_device_wrap.cpp @@ -165,8 +165,6 @@ HRESULT WrappedID3D11Device::CreateBuffer( SCOPED_SERIALISE_CONTEXT(CREATE_BUFFER); Serialise_CreateBuffer(pDesc, pInitialData, &wrapped); - scope.SetAlignment(32); - chunk = scope.Get(); } @@ -400,8 +398,6 @@ HRESULT WrappedID3D11Device::CreateTexture1D( SCOPED_SERIALISE_CONTEXT(CREATE_TEXTURE_1D); Serialise_CreateTexture1D(pDesc, pInitialData, &wrapped); - scope.SetAlignment(32); - chunk = scope.Get(); } @@ -503,8 +499,6 @@ HRESULT WrappedID3D11Device::CreateTexture2D( SCOPED_SERIALISE_CONTEXT(CREATE_TEXTURE_2D); Serialise_CreateTexture2D(pDesc, pInitialData, &wrapped); - scope.SetAlignment(32); - chunk = scope.Get(); } @@ -606,8 +600,6 @@ HRESULT WrappedID3D11Device::CreateTexture3D( SCOPED_SERIALISE_CONTEXT(CREATE_TEXTURE_3D); Serialise_CreateTexture3D(pDesc, pInitialData, &wrapped); - scope.SetAlignment(32); - chunk = scope.Get(); } @@ -2842,8 +2834,6 @@ HRESULT WrappedID3D11Device::OpenSharedResource( SCOPED_SERIALISE_CONTEXT(OPEN_SHARED_RESOURCE); Serialise_OpenSharedResource(hResource, ReturnedInterface, ppResource); - scope.SetAlignment(32); - chunk = scope.Get(); } diff --git a/renderdoc/driver/d3d11/d3d11_manager.h b/renderdoc/driver/d3d11/d3d11_manager.h index 7804e58b8..d3946e7b2 100644 --- a/renderdoc/driver/d3d11/d3d11_manager.h +++ b/renderdoc/driver/d3d11/d3d11_manager.h @@ -62,8 +62,8 @@ struct D3D11ResourceRecord : public ResourceRecord { if(ShadowPtr[ctx][0] == NULL) { - ShadowPtr[ctx][0] = Serialiser::AllocAlignedBuffer(size + sizeof(markerValue), 32); - ShadowPtr[ctx][1] = Serialiser::AllocAlignedBuffer(size + sizeof(markerValue), 32); + ShadowPtr[ctx][0] = Serialiser::AllocAlignedBuffer(size + sizeof(markerValue)); + ShadowPtr[ctx][1] = Serialiser::AllocAlignedBuffer(size + sizeof(markerValue)); memcpy(ShadowPtr[ctx][0] + size, markerValue, sizeof(markerValue)); memcpy(ShadowPtr[ctx][1] + size, markerValue, sizeof(markerValue)); diff --git a/renderdoc/driver/gl/gl_driver.cpp b/renderdoc/driver/gl/gl_driver.cpp index 47b35ec49..a5f6013ac 100644 --- a/renderdoc/driver/gl/gl_driver.cpp +++ b/renderdoc/driver/gl/gl_driver.cpp @@ -702,8 +702,6 @@ WrappedOpenGL::WrappedOpenGL(const char *logfile, const GLHookSet &funcs) m_TotalTime = m_AvgFrametime = m_MinFrametime = m_MaxFrametime = 0.0; - m_CurFileSize = 0; - m_RealDebugFunc = NULL; m_RealDebugFuncParam = NULL; @@ -2434,7 +2432,7 @@ bool WrappedOpenGL::EndFrameCapture(void *dev, void *wnd) RDCDEBUG("Done"); } - m_CurFileSize += m_pFileSerialiser->FlushToDisk(); + m_pFileSerialiser->FlushToDisk(); RenderDoc::Inst().SuccessfullyWrittenLog(); @@ -2953,7 +2951,7 @@ void WrappedOpenGL::ReadLogInitialisation() m_pSerialiser->SetDebugText(false); - m_pSerialiser->SetBase(firstFrame); + m_pSerialiser->SetPersistentBlock(firstFrame); } void WrappedOpenGL::ProcessChunk(uint64_t offset, GLChunkType context) diff --git a/renderdoc/driver/gl/gl_driver.h b/renderdoc/driver/gl/gl_driver.h index 05874caf7..7316c1f36 100644 --- a/renderdoc/driver/gl/gl_driver.h +++ b/renderdoc/driver/gl/gl_driver.h @@ -164,8 +164,6 @@ class WrappedOpenGL : public IFrameCapturer CaptureFailReason m_FailureReason; bool m_SuccessfulCapture; - uint64_t m_CurFileSize; - PerformanceTimer m_FrameTimer; vector m_FrameTimes; double m_TotalTime, m_AvgFrametime, m_MinFrametime, m_MaxFrametime; diff --git a/renderdoc/driver/gl/gl_resources.h b/renderdoc/driver/gl/gl_resources.h index af66b1088..df5ef2d51 100644 --- a/renderdoc/driver/gl/gl_resources.h +++ b/renderdoc/driver/gl/gl_resources.h @@ -184,12 +184,12 @@ struct GLResourceRecord : public ResourceRecord GLResource Resource; - void AllocShadowStorage(size_t size, size_t alignment = 16) + void AllocShadowStorage(size_t size) { if(ShadowPtr[0] == NULL) { - ShadowPtr[0] = Serialiser::AllocAlignedBuffer(size, alignment); - ShadowPtr[1] = Serialiser::AllocAlignedBuffer(size, alignment); + ShadowPtr[0] = Serialiser::AllocAlignedBuffer(size); + ShadowPtr[1] = Serialiser::AllocAlignedBuffer(size); } } diff --git a/renderdoc/driver/gl/wrappers/gl_buffer_funcs.cpp b/renderdoc/driver/gl/wrappers/gl_buffer_funcs.cpp index f8f893ef1..3190f384a 100644 --- a/renderdoc/driver/gl/wrappers/gl_buffer_funcs.cpp +++ b/renderdoc/driver/gl/wrappers/gl_buffer_funcs.cpp @@ -344,9 +344,6 @@ void WrappedOpenGL::Common_glNamedBufferStorageEXT(ResourceId id, GLsizeiptr siz SCOPED_SERIALISE_CONTEXT(BUFFERSTORAGE); Serialise_glNamedBufferStorageEXT(record->Resource.name, size, data, flags); - // for satisfying GL_MIN_MAP_BUFFER_ALIGNMENT - scope.SetAlignment(64); - Chunk *chunk = scope.Get(); { @@ -368,7 +365,7 @@ void WrappedOpenGL::Common_glNamedBufferStorageEXT(ResourceId id, GLsizeiptr siz RDCASSERT(record->Map.persistentPtr); // persistent maps always need both sets of shadow storage, so allocate up front. - record->AllocShadowStorage(size, 64); + record->AllocShadowStorage(size); } } else @@ -544,9 +541,6 @@ void WrappedOpenGL::glNamedBufferDataEXT(GLuint buffer, GLsizeiptr size, const v SCOPED_SERIALISE_CONTEXT(BUFFERDATA); Serialise_glNamedBufferDataEXT(buffer, size, data, usage); - // for satisfying GL_MIN_MAP_BUFFER_ALIGNMENT - scope.SetAlignment(64); - Chunk *chunk = scope.Get(); // if we've already created this is a renaming/data updating call. It should go in @@ -675,9 +669,6 @@ void WrappedOpenGL::glBufferData(GLenum target, GLsizeiptr size, const void *dat SCOPED_SERIALISE_CONTEXT(BUFFERDATA); Serialise_glNamedBufferDataEXT(buffer, size, data, usage); - // for satisfying GL_MIN_MAP_BUFFER_ALIGNMENT - scope.SetAlignment(64); - Chunk *chunk = scope.Get(); // if we've already created this is a renaming/data updating call. It should go in @@ -1752,7 +1743,7 @@ void *WrappedOpenGL::glMapNamedBufferRangeEXT(GLuint buffer, GLintptr offset, GL m_Real.glGetNamedBufferParameterivEXT(buffer, eGL_BUFFER_SIZE, &buflength); // allocate our shadow storage - record->AllocShadowStorage(buflength, 64); + record->AllocShadowStorage(buflength); shadow = (byte *)record->GetShadowPtr(0); // if we're not invalidating, we need the existing contents diff --git a/renderdoc/os/linux/linux_stringio.cpp b/renderdoc/os/linux/linux_stringio.cpp index 006ed3d50..a4ec891e7 100644 --- a/renderdoc/os/linux/linux_stringio.cpp +++ b/renderdoc/os/linux/linux_stringio.cpp @@ -345,6 +345,8 @@ namespace FileIO uint64_t ftell64(FILE *f) { return (uint64_t)::ftell(f); } void fseek64(FILE *f, uint64_t offset, int origin) { ::fseek(f, (long)offset, origin); } + bool feof(FILE *f) { return ::feof(f) != 0; } + int fclose(FILE *f) { return ::fclose(f); } }; diff --git a/renderdoc/os/os_specific.h b/renderdoc/os/os_specific.h index fbbbffa54..7db01c1cb 100644 --- a/renderdoc/os/os_specific.h +++ b/renderdoc/os/os_specific.h @@ -200,6 +200,8 @@ namespace FileIO uint64_t ftell64(FILE *f); void fseek64(FILE *f, uint64_t offset, int origin); + bool feof(FILE *f); + int fclose(FILE *f); }; diff --git a/renderdoc/os/win32/win32_shellext.cpp b/renderdoc/os/win32/win32_shellext.cpp index 7675fe419..fd53cef59 100644 --- a/renderdoc/os/win32/win32_shellext.cpp +++ b/renderdoc/os/win32/win32_shellext.cpp @@ -108,9 +108,9 @@ struct RDCThumbnailProvider : public IThumbnailProvider, IInitializeWithStream if(m_Inited) return HRESULT_FROM_WIN32(ERROR_ALREADY_INITIALIZED); - byte *buf = new byte[512*1024 + 1]; + byte *buf = new byte[2*1024*1024 + 1]; ULONG numRead = 0; - HRESULT hr = pstream->Read(buf, 512*1024, &numRead); + HRESULT hr = pstream->Read(buf, 2*1024*1024, &numRead); if(hr != S_OK && hr != S_FALSE) { @@ -122,6 +122,8 @@ struct RDCThumbnailProvider : public IThumbnailProvider, IInitializeWithStream m_Ser = new Serialiser(numRead, buf, true); + delete[] buf; + m_Inited = true; return S_OK; diff --git a/renderdoc/os/win32/win32_stringio.cpp b/renderdoc/os/win32/win32_stringio.cpp index ad6e03f68..db637c280 100644 --- a/renderdoc/os/win32/win32_stringio.cpp +++ b/renderdoc/os/win32/win32_stringio.cpp @@ -337,6 +337,8 @@ namespace FileIO uint64_t ftell64(FILE *f) { return ::_ftelli64(f); } void fseek64(FILE *f, uint64_t offset, int origin) { ::_fseeki64(f, offset, origin); } + bool feof(FILE *f) { return ::feof(f) != 0; } + int fclose(FILE *f) { return ::fclose(f); } }; diff --git a/renderdoc/serialise/serialiser.cpp b/renderdoc/serialise/serialiser.cpp index 480363489..0ce130c8d 100644 --- a/renderdoc/serialise/serialiser.cpp +++ b/renderdoc/serialise/serialiser.cpp @@ -28,6 +28,9 @@ #include "core/core.h" #include "serialise/string_utils.h" +#include "common/timing.h" + +#include "3rdparty/lz4/lz4.h" #ifdef _MSC_VER #pragma warning (disable : 4422) // warning C4422: 'snprintf' : too many arguments passed for format string @@ -43,9 +46,198 @@ int64_t Chunk::m_MaxChunks = 0; #endif const uint32_t Serialiser::MAGIC_HEADER = MAKE_FOURCC('R', 'D', 'O', 'C'); -const size_t Serialiser::BufferAlignment = 16; +const uint64_t Serialiser::BufferAlignment = 64; -Chunk::Chunk(Serialiser *ser, uint32_t chunkType, size_t alignment, bool temporary) +// based on blockStreaming_doubleBuffer.c in lz4 examples +struct CompressedFileIO +{ + // large block size + static const size_t BlockSize = 64 * 1024; + + CompressedFileIO(FILE *f) + { + m_F = f; + LZ4_resetStream(&m_LZ4Comp); + LZ4_setStreamDecode(&m_LZ4Decomp, NULL, 0); + m_CompressedSize = m_UncompressedSize = 0; + m_PageIdx = m_PageOffset = 0; + m_PageData = 0; + + m_CompressSize = LZ4_COMPRESSBOUND(BlockSize); + m_CompressBuf = new byte[m_CompressSize]; + } + + ~CompressedFileIO() + { + SAFE_DELETE_ARRAY(m_CompressBuf); + } + + uint32_t GetCompressedSize() { return m_CompressedSize; } + uint32_t GetUncompressedSize() { return m_UncompressedSize; } + + // write out some data - accumulate into the input pages, then + // when a page is full call Flush() to flush it out to disk + void Write(const void *data, size_t len) + { + if(data == NULL || len == 0) return; + + m_UncompressedSize += (uint32_t)len; + + const byte *src = (const byte *)data; + + size_t remainder = 0; + + // loop continually, writing up to BlockSize out of what remains of data + do + { + remainder = 0; + + // if we're about to copy more than the page, copy only + // what will fit, then copy the remainder after flushing + if(m_PageOffset + len > BlockSize) + { + remainder = len - (BlockSize - m_PageOffset); + len = BlockSize - m_PageOffset; + } + + memcpy(m_InPages[m_PageIdx] + m_PageOffset, src, len); + m_PageOffset += len; + + if(remainder > 0) + { + Flush(); // this will swap the input pages and reset the page offset + + src += len; + len = remainder; + } + } while(remainder > 0); + } + + // flush out the current page to disk + void Flush() + { + // m_PageOffset is the amount written, usually equal to BlockSize except the last block. + int32_t compSize = LZ4_compress_fast_continue(&m_LZ4Comp, (const char *)m_InPages[m_PageIdx], (char *)m_CompressBuf, (int)m_PageOffset, (int)m_CompressSize, 1); + + if(compSize < 0) + { + RDCERR("Error compressing: %i", compSize); + return; + } + + FileIO::fwrite(&compSize, sizeof(compSize), 1, m_F); + FileIO::fwrite(m_CompressBuf, 1, compSize, m_F); + + m_CompressedSize += compSize + sizeof(int32_t); + + m_PageOffset = 0; + m_PageIdx = 1 - m_PageIdx; + } + + // Reset back to 0, only makes sense when reading as writing can't be undone + void Reset() + { + LZ4_setStreamDecode(&m_LZ4Decomp, NULL, 0); + m_CompressedSize = m_UncompressedSize = 0; + m_PageIdx = 0; + m_PageOffset = 0; + } + + // read out some data - if the input page is empty we fill + // the next page with data from disk + void Read(byte *data, size_t len) + { + if(data == NULL || len == 0) return; + + m_UncompressedSize += (uint32_t)len; + + // loop continually, writing up to BlockSize out of what remains of data + do + { + size_t readamount = len; + + // if we're about to copy more than the page, copy only + // what will fit, then copy the remainder after refilling + if(readamount > m_PageData) + readamount = m_PageData; + + if(readamount > 0) + { + memcpy(data, m_InPages[m_PageIdx] + m_PageOffset, readamount); + + m_PageOffset += readamount; + m_PageData -= readamount; + + data += readamount; + len -= readamount; + } + + if(len > 0) + FillBuffer(); // this will swap the input pages and reset the page offset + } while(len > 0); + } + + void FillBuffer() + { + int32_t compSize = 0; + + FileIO::fread(&compSize, sizeof(compSize), 1, m_F); + FileIO::fread(m_CompressBuf, 1, compSize, m_F); + + m_CompressedSize += compSize; + + m_PageIdx = 1 - m_PageIdx; + + int32_t decompSize = LZ4_decompress_safe_continue(&m_LZ4Decomp, (const char *)m_CompressBuf, (char *)m_InPages[m_PageIdx], compSize, BlockSize); + + if(decompSize < 0) + { + RDCERR("Error decompressing: %i", decompSize); + return; + } + + m_PageOffset = 0; + m_PageData = decompSize; + } + + static void Decompress(byte *destBuf, const byte *srcBuf, size_t len) + { + LZ4_streamDecode_t lz4; + LZ4_setStreamDecode(&lz4, NULL, 0); + + const byte *srcBufEnd = srcBuf + len; + + while(srcBuf+4 < srcBufEnd) + { + const int32_t *compSize = (const int32_t *)srcBuf; + srcBuf = (const byte *)(compSize + 1); + + if(srcBuf + *compSize > srcBufEnd) + break; + + int32_t decompSize = LZ4_decompress_safe_continue(&lz4, (const char *)srcBuf, (char *)destBuf, *compSize, BlockSize); + + if(decompSize < 0) + return; + + srcBuf += *compSize; + destBuf += decompSize; + } + } + + LZ4_stream_t m_LZ4Comp; + LZ4_streamDecode_t m_LZ4Decomp; + FILE *m_F; + uint32_t m_CompressedSize, m_UncompressedSize; + + byte m_InPages[2][BlockSize]; + size_t m_PageIdx, m_PageOffset, m_PageData; + + byte *m_CompressBuf; + size_t m_CompressSize; +}; + +Chunk::Chunk(Serialiser *ser, uint32_t chunkType, bool temporary) { m_Length = (uint32_t)ser->GetOffset(); @@ -55,12 +247,7 @@ Chunk::Chunk(Serialiser *ser, uint32_t chunkType, size_t alignment, bool tempora m_Temporary = temporary; - if(alignment) - { - m_Data = Serialiser::AllocAlignedBuffer(m_Length, alignment); - m_AlignedData = true; - } - else if(ser->HasAlignedData()) + if(ser->HasAlignedData()) { m_Data = Serialiser::AllocAlignedBuffer(m_Length); m_AlignedData = true; @@ -110,77 +297,132 @@ Chunk::~Chunk() } } -void Serialiser::Reset() +/* + + ----------------------------- + File format for version 0x31: + + uint64_t MAGIC_HEADER; + uint64_t version = 0x00000031; + uint64_t filesize; + uint64_t resolveDBSize; + + if(resolveDBSize > 0) + { + byte resolveDB[resolveDBSize]; + byte paddingTo16Alignment[]; + } + + byte captureData[]; // remainder of the file + + ----------------------------- + File format for version 0x32: + + uint64_t MAGIC_HEADER; + uint64_t version = 0x00000032; + + 1 or more sections: + + Section + { + char isASCII = '\0' or 'A'; // indicates the section is ASCII or binary. ASCII allows for easy appending by hand/script + if(isASCII == 'A') + { + // ASCII sections are discouraged for tools, but useful for hand-editing by just + // appending a simple text file + char newline = '\n'; + char length[]; // length of just section data below, as decimal string + char newline = '\n'; + char sectionType[]; // section type, see SectionType enum, as decimal string. + char newline = '\n'; + char sectionName[]; // UTF-8 string name of section. + char newline = '\n'; + // sectionName is an arbitrary string. If two sections exist with the + // exact same name, the last one to occur in the file will be taken, the + // others ignored. + + byte sectiondata[ interpret(length) ]; // section data + } + else if(isASCII == '\0') + { + byte zero[3]; // pad out the above character with 0 bytes. Reserved for future use + uint32_t sectionFlags; // section flags - e.g. is compressed or not. + uint32_t sectionType; // section type enum, see SectionType. Could be eSectionType_Unknown + uint32_t sectionLength; // byte length of the actual section data + uint32_t sectionNameLength; // byte length of the string below (minimum 1, for null terminator) + char sectionName[sectionNameLength]; // UTF-8 string name of section, optional. + + byte sectiondata[length]; // actual contents of the section + + // note: compressed sections will contain the uncompressed length as a uint64_t + // before the compressed data. + } + }; + + // remainder of the file is tightly packed/unaligned section structures. + // The first section must always be the actual frame capture data in + // binary form + Section sections[]; + +*/ + +struct FileHeader { - if(m_ResolverThread != 0) + FileHeader() { - m_ResolverThreadKillSignal = true; - - Threading::JoinThread(m_ResolverThread); - Threading::CloseThread(m_ResolverThread); - m_ResolverThread = 0; + magic = Serialiser::MAGIC_HEADER; + version = Serialiser::SERIALISE_VERSION; } - m_DebugText = ""; + uint64_t magic; + uint64_t version; +}; - m_HasError = false; - m_ErrorCode = eSerError_None; +struct BinarySectionHeader +{ + byte isASCII; // 0x0 + byte zero[3]; // 0x0, 0x0, 0x0 + Serialiser::SectionFlags sectionFlags; // section flags - e.g. is compressed or not. + Serialiser::SectionType sectionType; // section type enum, see SectionType. Could be eSectionType_Unknown + uint32_t sectionLength; // byte length of the actual section data + uint32_t sectionNameLength; // byte length of the string below (could be 0) + char name[1]; // actually sectionNameLength, but at least 1 for null terminator - m_Mode = NONE; + // char name[sectionNameLength]; + // byte data[sectionLength]; +}; - m_Indent = 0; - - SAFE_DELETE_ARRAY(m_pCallstack); - SAFE_DELETE(m_pResolver); - if(m_Buffer) - { - FreeAlignedBuffer(m_Buffer); - m_Buffer = NULL; - } - - m_ChunkLookup = NULL; - - m_HasResolver = false; - - m_AlignedData = false; - - m_ReadFileHandle = NULL; - - m_Buffer = NULL; - m_BufferSize = 0; - m_BufferHead = NULL; -} +#define RETURNCORRUPT(...) { RDCERR(__VA_ARGS__); m_ErrorCode = eSerError_Corrupt; m_HasError = true; return; } Serialiser::Serialiser(size_t length, const byte *memoryBuf, bool fileheader) : m_pCallstack(NULL), m_pResolver(NULL), m_Buffer(NULL) { m_ResolverThread = 0; + // ensure binary sizes of enums + RDCCOMPILE_ASSERT(sizeof(SectionFlags) == sizeof(uint32_t), "Section flags not in uint32"); + RDCCOMPILE_ASSERT(sizeof(SectionType) == sizeof(uint32_t), "Section type not in uint32"); + + RDCCOMPILE_ASSERT(offsetof(BinarySectionHeader, name) == sizeof(uint32_t)*5, "BinarySectionHeader size has changed or contains padding"); + Reset(); - - m_DebugTextWriting = false; m_Mode = READING; m_DebugEnabled = false; if(!fileheader) { - m_HasResolver = false; - - m_FileStartOffset = 0; - m_BufferSize = length; m_CurrentBufferSize = (size_t)m_BufferSize; m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); - m_ReadOffset = 0; - memcpy(m_Buffer, memoryBuf + m_FileStartOffset, m_CurrentBufferSize); + memcpy(m_Buffer, memoryBuf, m_CurrentBufferSize); return; } - DebuggerHeader *header = (DebuggerHeader *)memoryBuf; + FileHeader *header = (FileHeader *)memoryBuf; - if(length < sizeof(DebuggerHeader)) + if(length < sizeof(FileHeader)) { RDCERR("Can't read from in-memory buffer, truncated header"); m_ErrorCode = eSerError_Corrupt; @@ -201,7 +443,129 @@ Serialiser::Serialiser(size_t length, const byte *memoryBuf, bool fileheader) return; } - if(header->version != SERIALISE_VERSION) + const byte *memoryBufEnd = memoryBuf + length; + + m_SerVer = header->version; + + if(header->version == 0x00000031) // backwards compatibility + { + memoryBuf += sizeof(FileHeader); + + if(length < sizeof(FileHeader) + sizeof(uint64_t)*2) + { + RDCERR("Can't read from in-memory buffer, truncated header"); + m_ErrorCode = eSerError_Corrupt; + m_HasError = true; + return; + } + + uint64_t *fileSize = (uint64_t *)memoryBuf; + memoryBuf += sizeof(uint64_t); + + uint64_t *resolveDBSize = (uint64_t *)memoryBuf; + memoryBuf += sizeof(uint64_t); + + if(*fileSize < length) + { + RDCERR("Overlong in-memory buffer. Expected length 0x016llx, got 0x016llx", *fileSize, length); + + m_ErrorCode = eSerError_Corrupt; + m_HasError = true; + return; + } + + // for in-memory case we don't need to load up the resolve db + + Section *frameCap = new Section(); + frameCap->type = eSectionType_FrameCapture; + frameCap->flags = eSectionFlag_None; + frameCap->fileoffset = 0; // irrelevant + frameCap->name = "renderdoc/internal/framecapture"; + frameCap->size = uint64_t(memoryBufEnd - memoryBuf); + + memoryBuf = AlignUpPtr(memoryBuf + *resolveDBSize, 16); + + m_Sections.push_back(frameCap); + m_KnownSections[eSectionType_FrameCapture] = frameCap; + } + else if(header->version == SERIALISE_VERSION) + { + const byte *memoryBufEnd = memoryBuf + length; + + memoryBuf += sizeof(FileHeader); + + // when loading in-memory we only care about the first section, which should be binary + const BinarySectionHeader *sectionHeader = (const BinarySectionHeader *)memoryBuf; + + // verify validity + if(memoryBuf + offsetof(BinarySectionHeader, name) >= memoryBufEnd) + { + RDCERR("Truncated binary section header"); + + m_ErrorCode = eSerError_Corrupt; + m_HasError = true; + return; + } + + if(sectionHeader->isASCII != 0 || + sectionHeader->zero[0] != 0 || + sectionHeader->zero[1] != 0 || + sectionHeader->zero[2] != 0) + { + RDCERR("Unexpected non-binary section first in capture when loading in-memory"); + + m_ErrorCode = eSerError_Corrupt; + m_HasError = true; + return; + } + + if(sectionHeader->sectionType != eSectionType_FrameCapture) + { + RDCERR("Expected first section to be frame capture, got type %x", sectionHeader->sectionType); + + m_ErrorCode = eSerError_Corrupt; + m_HasError = true; + return; + } + + memoryBuf += offsetof(BinarySectionHeader, name); + memoryBuf += sectionHeader->sectionNameLength; // skip name + + if(memoryBuf >= memoryBufEnd) + { + RDCERR("Truncated binary section header"); + + m_ErrorCode = eSerError_Corrupt; + m_HasError = true; + return; + } + + Section *frameCap = new Section(); + frameCap->fileoffset = 0; // irrelevant + frameCap->data.assign(memoryBuf, memoryBufEnd); + frameCap->name = sectionHeader->name; + frameCap->type = sectionHeader->sectionType; + frameCap->flags = sectionHeader->sectionFlags; + + uint64_t *uncompLength = (uint64_t *)memoryBuf; + + memoryBuf += sizeof(uint64_t); + + if(memoryBuf >= memoryBufEnd) + { + RDCERR("Truncated binary section header"); + + m_ErrorCode = eSerError_Corrupt; + m_HasError = true; + return; + } + + frameCap->size = *uncompLength; + + m_KnownSections[eSectionType_FrameCapture] = frameCap; + m_Sections.push_back(frameCap); + } + else { RDCERR("Capture file from wrong version. This program is on logfile version %llu, file is logfile version %llu", SERIALISE_VERSION, header->version); @@ -210,25 +574,18 @@ Serialiser::Serialiser(size_t length, const byte *memoryBuf, bool fileheader) return; } - if(header->fileSize < length) - { - RDCERR("Overlong in-memory buffer. Expected length 0x016llx, got 0x016llx", header->fileSize, length); - - m_ErrorCode = eSerError_Corrupt; - m_HasError = true; - return; - } - - m_HasResolver = header->resolveDBSize > 0; - - m_FileStartOffset = AlignUp16(sizeof(DebuggerHeader) + header->resolveDBSize); - - m_BufferSize = length-m_FileStartOffset; + m_BufferSize = m_KnownSections[eSectionType_FrameCapture]->size; m_CurrentBufferSize = (size_t)m_BufferSize; m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); - m_ReadOffset = 0; - memcpy(m_Buffer, memoryBuf + m_FileStartOffset, m_CurrentBufferSize); + if(m_KnownSections[eSectionType_FrameCapture]->flags & eSectionFlag_LZ4Compressed) + { + CompressedFileIO::Decompress(m_Buffer, memoryBuf, memoryBufEnd - memoryBuf); + } + else + { + memcpy(m_Buffer, memoryBuf, m_CurrentBufferSize); + } } Serialiser::Serialiser(const char *path, Mode mode, bool debugMode) @@ -239,13 +596,11 @@ Serialiser::Serialiser(const char *path, Mode mode, bool debugMode) Reset(); m_Filename = path ? path : ""; - - m_DebugTextWriting = false; m_Mode = mode; m_DebugEnabled = debugMode; - DebuggerHeader header; + FileHeader header; if(mode == READING) { @@ -261,7 +616,7 @@ Serialiser::Serialiser(const char *path, Mode mode, bool debugMode) RDCDEBUG("Opened capture file for read"); - FileIO::fread(&header, 1, sizeof(DebuggerHeader), m_ReadFileHandle); + FileIO::fread(&header, 1, sizeof(FileHeader), m_ReadFileHandle); if(header.magic != MAGIC_HEADER) { @@ -273,8 +628,201 @@ Serialiser::Serialiser(const char *path, Mode mode, bool debugMode) m_ReadFileHandle = 0; return; } - - if(header.version != SERIALISE_VERSION) + + m_SerVer = header.version; + + if(header.version == 0x00000031) // backwards compatibility + { + uint64_t headerRemainder[2]; + + FileIO::fread(&headerRemainder, 1, sizeof(headerRemainder), m_ReadFileHandle); + + FileIO::fseek64(m_ReadFileHandle, 0, SEEK_END); + + uint64_t realLength = FileIO::ftell64(m_ReadFileHandle); + if(headerRemainder[0] != realLength) + { + RDCERR("Truncated/overlong capture file. Expected length 0x016llx, got 0x016llx", headerRemainder[0], realLength); + + m_ErrorCode = eSerError_Corrupt; + m_HasError = true; + FileIO::fclose(m_ReadFileHandle); + m_ReadFileHandle = 0; + return; + } + + FileIO::fseek64(m_ReadFileHandle, sizeof(FileHeader) + sizeof(headerRemainder), SEEK_SET); + + size_t resolveDBSize = (size_t)headerRemainder[1]; + + if(resolveDBSize > 0) + { + Section *resolveDB = new Section(); + resolveDB->type = eSectionType_ResolveDatabase; + resolveDB->flags = eSectionFlag_None; + resolveDB->fileoffset = sizeof(FileHeader) + sizeof(headerRemainder); + resolveDB->name = "renderdoc/internal/resolvedb"; + resolveDB->data.resize(resolveDBSize); + + // read resolve DB entirely into memory + FileIO::fread(&resolveDB->data[0], 1, resolveDBSize, m_ReadFileHandle); + + m_Sections.push_back(resolveDB); + m_KnownSections[eSectionType_ResolveDatabase] = resolveDB; + } + + // seek to frame capture data + FileIO::fseek64(m_ReadFileHandle, AlignUp16(sizeof(FileHeader) + sizeof(headerRemainder) + resolveDBSize), SEEK_SET); + + Section *frameCap = new Section(); + frameCap->type = eSectionType_FrameCapture; + frameCap->flags = eSectionFlag_None; + frameCap->fileoffset = FileIO::ftell64(m_ReadFileHandle); + frameCap->name = "renderdoc/internal/framecapture"; + frameCap->size = realLength - frameCap->fileoffset; + + m_Sections.push_back(frameCap); + m_KnownSections[eSectionType_FrameCapture] = frameCap; + } + else if(header.version == SERIALISE_VERSION) + { + while(!FileIO::feof(m_ReadFileHandle)) + { + BinarySectionHeader sectionHeader = {0}; + byte *reading = (byte *)§ionHeader; + + FileIO::fread(reading, 1, 1, m_ReadFileHandle); reading++; + + if(FileIO::feof(m_ReadFileHandle)) + break; + + if(sectionHeader.isASCII == 'A') + { + // ASCII section + char c = 0; + FileIO::fread(&c, 1, 1, m_ReadFileHandle); + if(c != '\n') + RETURNCORRUPT("Invalid ASCII data section '%hhx'", c); + + if(FileIO::feof(m_ReadFileHandle)) + RETURNCORRUPT("Invalid truncated ASCII data section"); + + uint64_t length = 0; + + c = '0'; + + while(!FileIO::feof(m_ReadFileHandle) && c != '\n') + { + c = '0'; + FileIO::fread(&c, 1, 1, m_ReadFileHandle); + + if(c == '\n') break; + + length *= 10; + length += int(c - '0'); + } + + if(FileIO::feof(m_ReadFileHandle)) RETURNCORRUPT("Invalid truncated ASCII data section"); + + union + { + uint32_t u32; + SectionType t; + } type; + + type.u32 = 0; + + c = '0'; + + while(!FileIO::feof(m_ReadFileHandle) && c != '\n') + { + c = '0'; + FileIO::fread(&c, 1, 1, m_ReadFileHandle); + + if(c == '\n') break; + + type.u32 *= 10; + type.u32 += int(c - '0'); + } + + if(FileIO::feof(m_ReadFileHandle)) RETURNCORRUPT("Invalid truncated ASCII data section"); + + string name; + + c = 0; + + while(!FileIO::feof(m_ReadFileHandle) && c != '\n') + { + c = 0; + FileIO::fread(&c, 1, 1, m_ReadFileHandle); + + if(c == 0 || c == '\n') break; + + name.push_back(c); + } + + if(FileIO::feof(m_ReadFileHandle)) RETURNCORRUPT("Invalid truncated ASCII data section"); + + Section *sect = new Section(); + sect->flags = eSectionFlag_ASCIIStored; + sect->type = type.t; + sect->name = name; + sect->size = length; + sect->data.resize((size_t)length); + sect->fileoffset = FileIO::ftell64(m_ReadFileHandle); + + FileIO::fread(§->data[0], 1, (size_t)length, m_ReadFileHandle); + + if(sect->type != eSectionType_Unknown && sect->type < eSectionType_Num) + m_KnownSections[sect->type] = sect; + m_Sections.push_back(sect); + } + else if(sectionHeader.isASCII == 0x0) + { + FileIO::fread(reading, 1, offsetof(BinarySectionHeader, name)-1, m_ReadFileHandle); + + Section *sect = new Section(); + sect->flags = sectionHeader.sectionFlags; + sect->type = sectionHeader.sectionType; + sect->name.resize(sectionHeader.sectionNameLength-1); + sect->size = sectionHeader.sectionLength; + + FileIO::fread(§->name[0], 1, sectionHeader.sectionNameLength-1, m_ReadFileHandle); + char nullterm = 0; + FileIO::fread(&nullterm, 1, 1, m_ReadFileHandle); + + sect->fileoffset = FileIO::ftell64(m_ReadFileHandle); + + if(sect->flags & eSectionFlag_LZ4Compressed) + { + sect->compressedReader = new CompressedFileIO(m_ReadFileHandle); + FileIO::fread(§->size, 1, sizeof(uint64_t), m_ReadFileHandle); + + sect->fileoffset += sizeof(uint64_t); + } + + if(sect->type != eSectionType_Unknown && sect->type < eSectionType_Num) + m_KnownSections[sect->type] = sect; + m_Sections.push_back(sect); + + // if section isn't frame capture data and is small enough, read it all into memory now, otherwise skip + if(sect->type != eSectionType_FrameCapture && sectionHeader.sectionLength < 4*1024*1024) + { + sect->data.resize(sectionHeader.sectionLength); + FileIO::fread(§->data[0], 1, sectionHeader.sectionLength, m_ReadFileHandle); + } + else + { + FileIO::fseek64(m_ReadFileHandle, sectionHeader.sectionLength, SEEK_CUR); + } + } + else + { + RETURNCORRUPT("Unrecognised section type '%hhx'", sectionHeader.isASCII); + } + } + } + else { RDCERR("Capture file from wrong version. This program is logfile version %llu, file is logfile version %llu", SERIALISE_VERSION, header.version); @@ -284,13 +832,10 @@ Serialiser::Serialiser(const char *path, Mode mode, bool debugMode) m_ReadFileHandle = 0; return; } - - FileIO::fseek64(m_ReadFileHandle, 0, SEEK_END); - - uint64_t realLength = FileIO::ftell64(m_ReadFileHandle); - if(header.fileSize != realLength) + + if(m_KnownSections[eSectionType_FrameCapture] == NULL) { - RDCERR("Truncated/overlong capture file. Expected length 0x016llx, got 0x016llx", header.fileSize, realLength); + RDCERR("Capture file doesn't have a frame capture"); m_ErrorCode = eSerError_Corrupt; m_HasError = true; @@ -299,20 +844,19 @@ Serialiser::Serialiser(const char *path, Mode mode, bool debugMode) return; } - m_HasResolver = header.resolveDBSize > 0; - - m_FileStartOffset = AlignUp16(sizeof(DebuggerHeader) + header.resolveDBSize); - - m_BufferSize = realLength-m_FileStartOffset; + m_BufferSize = m_KnownSections[eSectionType_FrameCapture]->size; m_CurrentBufferSize = (size_t)RDCMIN(m_BufferSize, (uint64_t)64*1024); m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); m_ReadOffset = 0; + FileIO::fseek64(m_ReadFileHandle, m_KnownSections[eSectionType_FrameCapture]->fileoffset, SEEK_SET); + + // read initial buffer of data ReadFromFile(0, m_CurrentBufferSize); } else { - m_pResolver = NULL; + m_SerVer = SERIALISE_VERSION; if(m_Filename != "") { @@ -324,21 +868,209 @@ Serialiser::Serialiser(const char *path, Mode mode, bool debugMode) m_BufferSize = 128*1024; m_BufferHead = m_Buffer = AllocAlignedBuffer((size_t)m_BufferSize); } - - m_ReadOffset = 0; - m_FileStartOffset = 0; } } -void Serialiser::ReadFromFile(uint64_t destOffs, size_t chunkLen) +void Serialiser::Reset() +{ + if(m_ResolverThread != 0) + { + m_ResolverThreadKillSignal = true; + + Threading::JoinThread(m_ResolverThread); + Threading::CloseThread(m_ResolverThread); + m_ResolverThread = 0; + } + + m_DebugText = ""; + m_DebugTextWriting = false; + + RDCEraseEl(m_KnownSections); + + m_HasError = false; + m_ErrorCode = eSerError_None; + + m_Mode = NONE; + + m_Indent = 0; + + SAFE_DELETE(m_pCallstack); + SAFE_DELETE(m_pResolver); + if(m_Buffer) + { + FreeAlignedBuffer(m_Buffer); + m_Buffer = NULL; + } + + m_ChunkLookup = NULL; + + m_AlignedData = false; + + m_ReadFileHandle = NULL; + + m_ReadOffset = 0; + + m_BufferHead = m_Buffer = NULL; + m_CurrentBufferSize = 0; + m_BufferSize = 0; +} + +Serialiser::~Serialiser() +{ + if(m_ResolverThread != 0) + { + m_ResolverThreadKillSignal = true; + Threading::JoinThread(m_ResolverThread); + Threading::CloseThread(m_ResolverThread); + m_ResolverThread = 0; + } + + if(m_ReadFileHandle) + { + FileIO::fclose(m_ReadFileHandle); + m_ReadFileHandle = 0; + } + + for(size_t i=0; i < m_Sections.size(); i++) + SAFE_DELETE(m_Sections[i]->compressedReader); + + for(size_t i=0; i < m_Chunks.size(); i++) + { + if(m_Chunks[i]->IsTemporary()) + SAFE_DELETE(m_Chunks[i]); + } + + m_Chunks.clear(); + + SAFE_DELETE(m_pResolver); + SAFE_DELETE(m_pCallstack); + if(m_Buffer) + { + FreeAlignedBuffer(m_Buffer); + m_Buffer = NULL; + } + m_Buffer = NULL; + m_BufferHead = NULL; +} + +void Serialiser::WriteBytes( const byte *buf, size_t nBytes ) +{ +#ifdef DEBUG_TEXT_SERIALISER + if(m_Mode == DEBUGWRITING) + return; +#endif + + if(m_HasError) + { + RDCERR("Writing bytes with error state serialiser"); + return; + } + + if(m_Buffer+m_BufferSize < m_BufferHead+nBytes+8) + { + // reallocate + while(m_Buffer+m_BufferSize < m_BufferHead+nBytes+8) + { + m_BufferSize += 128*1024; + } + + byte *newBuf = AllocAlignedBuffer((size_t)m_BufferSize); + + size_t curUsed = m_BufferHead-m_Buffer; + + memcpy(newBuf, m_Buffer, curUsed); + + FreeAlignedBuffer(m_Buffer); + + m_Buffer = newBuf; + m_BufferHead = newBuf + curUsed; + } + + memcpy(m_BufferHead, buf, nBytes); + + m_BufferHead += nBytes; +} + +void * Serialiser::ReadBytes( size_t nBytes ) +{ + if(m_HasError) + { + RDCERR("Reading bytes with error state serialiser"); + return NULL; + } + + // if we would read off the end of our current window + if(m_BufferHead+nBytes > m_Buffer+m_CurrentBufferSize) + { + // store old buffer and the read data, so we can move it into the new buffer + byte *oldBuffer = m_Buffer; + + // always keep at least a certain window behind what we read. + size_t backwardsWindow = RDCMIN((size_t)64, size_t(m_BufferHead - m_Buffer)); + + byte *currentData = m_BufferHead - backwardsWindow; + size_t currentDataSize = m_CurrentBufferSize - (m_BufferHead-m_Buffer) + backwardsWindow; + + size_t BufferOffset = m_BufferHead-m_Buffer; + + // if we are reading more than our current buffer size, expand the buffer size + if(nBytes+backwardsWindow > m_CurrentBufferSize) + { + // very conservative resizing - don't do "double and add" - to avoid + // a 1GB buffer being read and needing to allocate 2GB. The cost is we + // will reallocate a bit more often + m_CurrentBufferSize = nBytes+backwardsWindow; + m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); + } + + // move the unread data into the buffer + memmove(m_Buffer, currentData, currentDataSize); + + if(BufferOffset > backwardsWindow) + { + m_ReadOffset += BufferOffset-backwardsWindow; + m_BufferHead = m_Buffer+backwardsWindow; + } + else + { + m_BufferHead = m_Buffer+BufferOffset; + } + + // if there's anything left of the file to read in, do so now + ReadFromFile(currentDataSize, RDCMIN(m_CurrentBufferSize-currentDataSize, size_t(m_BufferSize - m_ReadOffset - currentDataSize))); + + if(oldBuffer != m_Buffer) + FreeAlignedBuffer(oldBuffer); + } + + void *ret = m_BufferHead; + + m_BufferHead += nBytes; + + RDCASSERT(m_BufferHead <= m_Buffer+m_CurrentBufferSize); + + return ret; +} + +void Serialiser::ReadFromFile(uint64_t bufferOffs, size_t length) { RDCASSERT(m_ReadFileHandle); if(m_ReadFileHandle == NULL) return; - FileIO::fseek64(m_ReadFileHandle, m_FileStartOffset+destOffs, SEEK_SET); - FileIO::fread(m_Buffer + destOffs - m_ReadOffset, 1, chunkLen, m_ReadFileHandle); + Section *s = m_KnownSections[eSectionType_FrameCapture]; + + RDCASSERT(s); + + if(s->flags & eSectionFlag_LZ4Compressed) + { + s->compressedReader->Read(m_Buffer + bufferOffs, length); + } + else + { + FileIO::fread(m_Buffer + bufferOffs, 1, length, m_ReadFileHandle); + } } byte *Serialiser::AllocAlignedBuffer(size_t size, size_t alignment) @@ -381,9 +1113,80 @@ void Serialiser::FreeAlignedBuffer(byte *buf) delete[] rawAlloc; } +void Serialiser::SetPersistentBlock(uint64_t offs) +{ + FreeAlignedBuffer(m_Buffer); + + RDCASSERT(m_BufferSize - offs < 0xffffffff); + + m_CurrentBufferSize = (size_t)(m_BufferSize - offs); + m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); + m_ReadOffset = offs; + + RDCASSERT(m_ReadFileHandle); + + Section *s = m_KnownSections[eSectionType_FrameCapture]; + RDCASSERT(s); + FileIO::fseek64(m_ReadFileHandle, s->fileoffset, SEEK_SET); + + s->compressedReader->Reset(); + + // can't seek arbitrarily in the stream, need to read through the rest + while(offs > 0) + { + // read at most buffer size at a time. + size_t chunkSize = RDCMIN(m_CurrentBufferSize, (size_t)offs); + ReadFromFile(0, chunkSize); + offs -= chunkSize; + } + + // now read the rest + ReadFromFile(0, m_CurrentBufferSize); + FileIO::fclose(m_ReadFileHandle); + m_ReadFileHandle = 0; +} + +void Serialiser::SetOffset(uint64_t offs) +{ + if(m_HasError) + { + RDCERR("Setting offset with error state serialiser"); + return; + } + + // if we're jumping back before our in-memory window just reset the window + // and load it all in from scratch. + if(m_Mode == READING && offs < m_ReadOffset) + { + // if we're reading from file, only support rewinding all the way to the start + RDCASSERT(m_ReadFileHandle == NULL || offs == 0); + + if(m_ReadFileHandle) + { + Section *s = m_KnownSections[eSectionType_FrameCapture]; + RDCASSERT(s); + FileIO::fseek64(m_ReadFileHandle, s->fileoffset, SEEK_SET); + + s->compressedReader->Reset(); + } + + FreeAlignedBuffer(m_Buffer); + + m_CurrentBufferSize = (size_t)RDCMIN(m_BufferSize, (uint64_t)64*1024); + m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); + m_ReadOffset = offs; + + ReadFromFile(0, m_CurrentBufferSize); + } + + RDCASSERT(m_BufferHead && m_Buffer && offs <= GetSize()); + m_BufferHead = m_Buffer + offs - m_ReadOffset; + m_Indent = 0; +} + void Serialiser::InitCallstackResolver() { - if(m_pResolver == NULL && m_ResolverThread == 0) + if(m_pResolver == NULL && m_ResolverThread == 0 && m_KnownSections[eSectionType_ResolveDatabase] != NULL) { m_ResolverThreadKillSignal = false; m_ResolverThread = Threading::CreateThread(&Serialiser::CreateResolver, (void *)this); @@ -401,69 +1204,19 @@ void Serialiser::SetCallstack(uint64_t *levels, size_t numLevels) void Serialiser::CreateResolver(void *ths) { Serialiser *ser = (Serialiser *)ths; - FILE *binFile = FileIO::fopen(ser->m_Filename.c_str(), "rb"); - if(!binFile) - { - RDCERR("Can't open capture file '%s' for read - errno %d", ser->m_Filename.c_str(), errno); - return; - } - - DebuggerHeader header; - FileIO::fread(&header, 1, sizeof(DebuggerHeader), binFile); - - if(header.magic != MAGIC_HEADER) - { - RDCERR("Invalid capture file. Expected 0x%08lx, got 0x%08llx", MAGIC_HEADER, header.magic); - FileIO::fclose(binFile); - return; - } - - if(header.version != SERIALISE_VERSION) - { - RDCERR("Capture file from wrong version. This program is logfile version %llu, file is logfile version %llu", SERIALISE_VERSION, header.version); - FileIO::fclose(binFile); - return; - } - - FileIO::fseek64(binFile, 0, SEEK_END); - - uint64_t realLength = FileIO::ftell64(binFile); - if(header.fileSize != realLength) - { - RDCERR("Truncated/overlong capture file. Expected length 0x08llx, got 0x08lx", header.fileSize, realLength); - FileIO::fclose(binFile); - return; - } - - if(header.resolveDBSize == 0) - { - RDCWARN("Trying to create resolver when no resolve DB is in the capture file"); - FileIO::fclose(binFile); - return; - } - - FileIO::fseek64(binFile, sizeof(DebuggerHeader), SEEK_SET); - - RDCASSERT(header.resolveDBSize < 0xffffffff); - - char *resolveDB = new char[(size_t)header.resolveDBSize]; - - FileIO::fread(resolveDB, 1, (size_t)header.resolveDBSize, binFile); - - FileIO::fclose(binFile); - string dir = dirname(ser->m_Filename); - Callstack::StackResolver *resolver = Callstack::MakeResolver(resolveDB, (size_t)header.resolveDBSize, dir, &ser->m_ResolverThreadKillSignal); + Section *s = ser->m_KnownSections[Serialiser::eSectionType_ResolveDatabase]; + RDCASSERT(s); - ser->m_pResolver = resolver; - - SAFE_DELETE_ARRAY(resolveDB); + ser->m_pResolver = Callstack::MakeResolver((char *)&s->data[0], s->data.size(), dir, &ser->m_ResolverThreadKillSignal); } -uint64_t Serialiser::FlushToDisk() +void Serialiser::FlushToDisk() { + SCOPED_TIMER("File writing"); + if(m_Filename != "" && !m_HasError && m_Mode == WRITING) { RDCDEBUG("writing capture files"); @@ -484,20 +1237,124 @@ uint64_t Serialiser::FlushToDisk() } } - FILE *binFile = FileIO::fopen(m_Filename.c_str(), "wb"); + FILE *binFile = FileIO::fopen(m_Filename.c_str(), "w+b"); if(!binFile) { RDCERR("Can't open capture file '%s' for write, errno %d", m_Filename.c_str(), errno); m_ErrorCode = eSerError_FileIO; m_HasError = true; - return 0; + return; } RDCDEBUG("Opened capture file for write"); - DebuggerHeader header; + FileHeader header; // automagically initialised with correct data + + // write header + FileIO::fwrite(&header, 1, sizeof(FileHeader), binFile); + + static const byte padding[BufferAlignment] = {0}; + uint64_t compressedSizeOffset = 0; + uint64_t uncompressedSizeOffset = 0; + + // write frame capture section header + { + const char sectionName[] = "renderdoc/internal/framecapture"; + + BinarySectionHeader section = { 0 }; + section.isASCII = 0; // redundant but explicit + section.sectionNameLength = sizeof(sectionName); // includes null terminator + section.sectionType = eSectionType_FrameCapture; + section.sectionFlags = eSectionFlag_LZ4Compressed; + section.sectionLength = 0; // will be fixed up later, to avoid having to compress everything into memory + + compressedSizeOffset = FileIO::ftell64(binFile) + offsetof(BinarySectionHeader, sectionLength); + + FileIO::fwrite(§ion, 1, offsetof(BinarySectionHeader, name), binFile); + FileIO::fwrite(sectionName, 1, sizeof(sectionName), binFile); + + uint64_t len = 0; // will be fixed up later + uncompressedSizeOffset = FileIO::ftell64(binFile); + FileIO::fwrite(&len, 1, sizeof(uint64_t), binFile); + } + + CompressedFileIO fwriter(binFile); + + // track offset so we can add padding. The padding is relative + // to the start of the decompressed buffer, so we start it from 0 + uint64_t offs = 0; + uint64_t alignedoffs = 0; + + // write frame capture contents + for(size_t i=0; i < m_Chunks.size(); i++) + { + Chunk *chunk = m_Chunks[i]; + + alignedoffs = AlignUp(offs, BufferAlignment); + + if(offs != alignedoffs && chunk->IsAligned()) + { + uint16_t chunkIdx = 0; // write a '0' chunk that indicates special behaviour + fwriter.Write(&chunkIdx, sizeof(chunkIdx)); + offs += sizeof(chunkIdx); + + uint8_t controlByte = 0; // control byte 0 indicates padding + fwriter.Write(&controlByte, sizeof(controlByte)); + offs += sizeof(controlByte); + + offs++; // we will have to write out a byte indicating how much padding exists, so add 1 + alignedoffs = AlignUp(offs, BufferAlignment); + + RDCCOMPILE_ASSERT(BufferAlignment < 0x100, "Buffer alignment must be less than 256"); // with a byte at most indicating how many bytes to pad, + // this is our maximal representable alignment + + uint8_t padLength = (alignedoffs-offs)&0xff; + fwriter.Write(&padLength, sizeof(padLength)); + + // we might have padded with the control bytes, so only write some bytes if we need to + if(padLength > 0) + { + fwriter.Write(padding, size_t(alignedoffs-offs)); + offs += alignedoffs-offs; + } + } + + fwriter.Write(chunk->GetData(), chunk->GetLength()); + + offs += chunk->GetLength(); + + if(chunk->IsTemporary()) + SAFE_DELETE(chunk); + } + + fwriter.Flush(); + + m_Chunks.clear(); + + // fixup section size + { + uint32_t compsize = 0; + uint64_t uncompsize = 0; + + uint64_t curoffs = FileIO::ftell64(binFile); + + FileIO::fseek64(binFile, compressedSizeOffset, SEEK_SET); + + compsize = fwriter.GetCompressedSize(); + FileIO::fwrite(&compsize, 1, sizeof(compsize), binFile); + + FileIO::fseek64(binFile, uncompressedSizeOffset, SEEK_SET); + + uncompsize = fwriter.GetUncompressedSize(); + FileIO::fwrite(&uncompsize, 1, sizeof(uncompsize), binFile); + + FileIO::fseek64(binFile, curoffs, SEEK_SET); + + RDCLOG("Compressed frame capture data from %u to %u", fwriter.GetUncompressedSize(), fwriter.GetCompressedSize()); + } + char *symbolDB = NULL; size_t symbolDBSize = 0; @@ -513,118 +1370,28 @@ uint64_t Serialiser::FlushToDisk() Callstack::GetLoadedModules(symbolDB, symbolDBSize); } - header.resolveDBSize = symbolDBSize; - - // write header - FileIO::fwrite(&header, 1, sizeof(DebuggerHeader), binFile); - - // write symbol database - if(symbolDBSize > 0) - FileIO::fwrite(symbolDB, 1, symbolDBSize, binFile); - - static const byte padding[BufferAlignment] = {0}; - - size_t offs = sizeof(DebuggerHeader) + symbolDBSize; - size_t alignedoffs = AlignUp16(offs); - - FileIO::fwrite(padding, 1, alignedoffs-offs, binFile); - - offs = alignedoffs; - - // write serialise contents - for(size_t i=0; i < m_Chunks.size(); i++) + // write symbol database section + if(symbolDB) { - Chunk *chunk = m_Chunks[i]; + const char sectionName[] = "renderdoc/internal/resolvedb"; - alignedoffs = AlignUp16(offs); - - if(offs != alignedoffs && chunk->IsAligned()) - { - uint16_t chunkIdx = 0; // write a '0' chunk that indicates special behaviour - FileIO::fwrite(&chunkIdx, sizeof(chunkIdx), 1, binFile); - offs += sizeof(chunkIdx); - - uint8_t controlByte = 0; // control byte 0 indicates padding - FileIO::fwrite(&controlByte, sizeof(controlByte), 1, binFile); - offs += sizeof(controlByte); - - offs++; // we will have to write out a byte indicating how much padding exists, so add 1 - alignedoffs = AlignUp16(offs); - - RDCCOMPILE_ASSERT(BufferAlignment < 0x100, "Buffer alignment must be less than 256"); // with a byte at most indicating how many bytes to pad, - // this is our maximal representable alignment - - uint8_t padLength = (alignedoffs-offs)&0xff; - FileIO::fwrite(&padLength, sizeof(padLength), 1, binFile); - - // we might have padded with the control bytes, so only write some bytes if we need to - if(padLength > 0) - { - FileIO::fwrite(padding, 1, alignedoffs-offs, binFile); - offs += alignedoffs-offs; - } - } + BinarySectionHeader section = { 0 }; + section.isASCII = 0; // redundant but explicit + section.sectionNameLength = sizeof(sectionName); // includes null terminator + section.sectionType = eSectionType_ResolveDatabase; + section.sectionLength = (uint32_t)symbolDBSize; - FileIO::fwrite(chunk->GetData(), 1, chunk->GetLength(), binFile); + FileIO::fwrite(§ion, 1, offsetof(BinarySectionHeader, name), binFile); + FileIO::fwrite(sectionName, 1, sizeof(sectionName), binFile); - offs += chunk->GetLength(); + // write actual data + FileIO::fwrite(symbolDB, 1, symbolDBSize, binFile); - if(chunk->IsTemporary()) - SAFE_DELETE(chunk); + SAFE_DELETE_ARRAY(symbolDB); } - m_Chunks.clear(); - - header.fileSize = offs; - - FileIO::fseek64(binFile, 0, SEEK_SET); - - // write header with correct filesize (accounting for all padding) - FileIO::fwrite(&header, 1, sizeof(DebuggerHeader), binFile); - FileIO::fclose(binFile); - - SAFE_DELETE_ARRAY(symbolDB); - - return header.fileSize; } - - return 0; -} - -Serialiser::~Serialiser() -{ - if(m_ResolverThread != 0) - { - m_ResolverThreadKillSignal = true; - Threading::JoinThread(m_ResolverThread); - Threading::CloseThread(m_ResolverThread); - m_ResolverThread = 0; - } - - if(m_ReadFileHandle) - { - FileIO::fclose(m_ReadFileHandle); - m_ReadFileHandle = 0; - } - - for(size_t i=0; i < m_Chunks.size(); i++) - { - if(m_Chunks[i]->IsTemporary()) - SAFE_DELETE(m_Chunks[i]); - } - - m_Chunks.clear(); - - SAFE_DELETE(m_pResolver); - SAFE_DELETE(m_pCallstack); - if(m_Buffer) - { - FreeAlignedBuffer(m_Buffer); - m_Buffer = NULL; - } - m_Buffer = NULL; - m_BufferHead = NULL; } void Serialiser::DebugPrint(const char *fmt, ...) @@ -921,27 +1688,12 @@ void Serialiser::Insert(Chunk *chunk) m_DebugText += chunk->GetDebugString(); } -void Serialiser::SkipBuffer() -{ - RDCASSERT(m_Mode < WRITING); - - uint32_t len; - ReadInto(len); - - // ensure byte alignment - uint64_t offs = GetOffset(); - uint64_t alignedoffs = AlignUp16(offs); - - if(offs != alignedoffs) - { - ReadBytes((size_t)(alignedoffs-offs)); - } - - ReadBytes(len); -} - void Serialiser::AlignNextBuffer(const size_t alignment) { + // on new logs, we don't have to align. This code will be deleted once backwards-compat is dropped + if(m_Mode >= WRITING || m_SerVer >= 0x00000032) + return; + // this is a super hack but it's the easiest way to align a buffer to a larger pow2 alignment // than the default 16-bytes, while still able to be backwards compatible with old logs that // weren't so aligned. We know that SerialiseBuffer will align to the nearest 16-byte boundary @@ -991,7 +1743,7 @@ void Serialiser::SerialiseBuffer(const char *name, byte *&buf, size_t &len) // ensure byte alignment uint64_t offs = GetOffset(); - uint64_t alignedoffs = AlignUp16(offs); + uint64_t alignedoffs = AlignUp(offs, BufferAlignment); if(offs != alignedoffs) { @@ -1011,7 +1763,9 @@ void Serialiser::SerialiseBuffer(const char *name, byte *&buf, size_t &len) // ensure byte alignment uint64_t offs = GetOffset(); - uint64_t alignedoffs = AlignUp16(offs); + + // serialise version 0x00000031 had only 16-byte alignment + uint64_t alignedoffs = AlignUp(offs, m_SerVer == 0x00000031 ? 16 : BufferAlignment); if(offs != alignedoffs) { diff --git a/renderdoc/serialise/serialiser.h b/renderdoc/serialise/serialiser.h index ff8b6310c..7525f5c65 100644 --- a/renderdoc/serialise/serialiser.h +++ b/renderdoc/serialise/serialiser.h @@ -80,6 +80,7 @@ typedef const char *(*ChunkLookup)(uint32_t chunkType); class Serialiser; class ScopedContext; +struct CompressedFileIO; // holds the memory, length and type for a given chunk, so that it can be // passed around and moved between owners before being serialised out @@ -105,7 +106,7 @@ class Chunk #endif // grab current contents of the serialiser into this chunk - Chunk(Serialiser *ser, uint32_t chunkType, size_t alignment, bool temp); + Chunk(Serialiser *ser, uint32_t chunkType, bool temp); private: // no copy semantics @@ -162,10 +163,28 @@ class Serialiser eSerError_UnsupportedVersion, }; + enum SectionFlags + { + eSectionFlag_None = 0x0, + eSectionFlag_ASCIIStored = 0x1, + eSectionFlag_LZ4Compressed = 0x2, + }; + + enum SectionType + { + eSectionType_Unknown = 0, + eSectionType_FrameCapture, // renderdoc/internal/framecapture + eSectionType_ResolveDatabase, // renderdoc/internal/resolvedb + eSectionType_FrameBookmarks, // renderdoc/ui/bookmarks + eSectionType_Notes, // renderdoc/ui/notes + eSectionType_Num, + }; + // version number of overall file format or chunk organisation. If the contents/meaning/order of // chunks have changed this does not need to be bumped, there are version numbers within each // API that interprets the stream that can be bumped. - static const uint64_t SERIALISE_VERSION = 0x00000031; + static const uint64_t SERIALISE_VERSION = 0x00000032; + static const uint32_t MAGIC_HEADER; ////////////////////////////////////////// // Init and error handling @@ -228,46 +247,9 @@ class Serialiser // Set up the base pointer and size. Serialiser will allocate enough for // the rest of the file and keep it all in memory (useful to keep everything // in actual frame data resident in memory). - void SetBase(uint64_t offs) - { - FreeAlignedBuffer(m_Buffer); + void SetPersistentBlock(uint64_t offs); - RDCASSERT(m_BufferSize - offs < 0xffffffff); - - m_CurrentBufferSize = (size_t)(m_BufferSize - offs); - m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); - m_ReadOffset = offs; - - ReadFromFile(offs, m_CurrentBufferSize); - FileIO::fclose(m_ReadFileHandle); - m_ReadFileHandle = 0; - } - - void SetOffset(uint64_t offs) - { - if(m_HasError) - { - RDCERR("Setting offset with error state serialiser"); - return; - } - - // if we're jumping back before our in-memory window just reset the window - // and load it all in from scratch. - if(m_Mode == READING && offs < m_ReadOffset) - { - FreeAlignedBuffer(m_Buffer); - - m_CurrentBufferSize = (size_t)RDCMIN(m_BufferSize, (uint64_t)64*1024); - m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); - m_ReadOffset = offs; - - ReadFromFile(offs, m_CurrentBufferSize); - } - - RDCASSERT(m_BufferHead && m_Buffer && offs <= GetSize()); - m_BufferHead = m_Buffer + offs - m_ReadOffset; - m_Indent = 0; - } + void SetOffset(uint64_t offs); void Rewind() { @@ -311,7 +293,7 @@ class Serialiser } void InitCallstackResolver(); - bool HasCallstacks() { return m_HasResolver; } + bool HasCallstacks() { return m_KnownSections[eSectionType_ResolveDatabase] != NULL; } // get callstack resolver, created with the DB in the file Callstack::StackResolver *GetCallstackResolver() @@ -553,7 +535,6 @@ class Serialiser // If serialising in, buf must either be NULL in which case allocated // memory will be returned, or it must be already large enough. void SerialiseBuffer(const char *name, byte *&buf, size_t &len); - void SkipBuffer(); void AlignNextBuffer(const size_t alignment); // NOT recommended interface. Useful for specific situations if e.g. you have @@ -573,10 +554,10 @@ class Serialiser // prints to the debug output log void DebugPrint(const char *fmt, ...); - static byte *AllocAlignedBuffer(size_t size, size_t align = 16); + static byte *AllocAlignedBuffer(size_t size, size_t align = 64); static void FreeAlignedBuffer(byte *buf); - uint64_t FlushToDisk(); + void FlushToDisk(); // set a function used when serialising a text representation // of the chunks @@ -616,87 +597,10 @@ class Serialiser ////////////////////////////////////////// // Raw memory buffer read/write - void WriteBytes(const byte *buf, size_t nBytes) - { -#ifdef DEBUG_TEXT_SERIALISER - if(m_Mode == DEBUGWRITING) - return; -#endif + void WriteBytes(const byte *buf, size_t nBytes); + void *ReadBytes(size_t nBytes); - if(m_HasError) - { - RDCERR("Writing bytes with error state serialiser"); - return; - } - - if(m_Buffer+m_BufferSize < m_BufferHead+nBytes+8) - { - // reallocate - while(m_Buffer+m_BufferSize < m_BufferHead+nBytes+8) - { - m_BufferSize += 128*1024; - } - - byte *newBuf = AllocAlignedBuffer((size_t)m_BufferSize); - - size_t curUsed = m_BufferHead-m_Buffer; - - memcpy(newBuf, m_Buffer, curUsed); - - FreeAlignedBuffer(m_Buffer); - - m_Buffer = newBuf; - m_BufferHead = newBuf + curUsed; - } - - memcpy(m_BufferHead, buf, nBytes); - - m_BufferHead += nBytes; - } - void *ReadBytes(size_t nBytes) - { - if(m_HasError) - { - RDCERR("Reading bytes with error state serialiser"); - return NULL; - } - - // if we would read off the end of our current window - if(m_BufferHead+nBytes > m_Buffer+m_CurrentBufferSize) - { - size_t BufferOffset = m_BufferHead-m_Buffer; - - if(nBytes+64 > m_CurrentBufferSize) - { - FreeAlignedBuffer(m_Buffer); - m_CurrentBufferSize = nBytes+64; - m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize); - } - - if(BufferOffset > 64) - { - m_ReadOffset += BufferOffset-64; - m_BufferHead = m_Buffer+64; - } - else - { - m_BufferHead = m_Buffer+BufferOffset; - } - - // if there's anything left of the file to read in, do so now - ReadFromFile(m_ReadOffset, RDCMIN(m_CurrentBufferSize, (size_t)(m_BufferSize-m_ReadOffset))); - } - - void *ret = m_BufferHead; - - m_BufferHead += nBytes; - - RDCASSERT(m_BufferHead <= m_Buffer+m_CurrentBufferSize); - - return ret; - } - - void ReadFromFile(uint64_t destOffs, size_t chunkLen); + void ReadFromFile(uint64_t bufferOffs, size_t length); template void WriteFrom(const T &f) { @@ -733,25 +637,11 @@ class Serialiser ////////////////////////////////////////// - static const uint32_t MAGIC_HEADER; - - static const size_t BufferAlignment; - - struct DebuggerHeader - { - DebuggerHeader() - { - magic = MAGIC_HEADER; - version = SERIALISE_VERSION; - } - - uint64_t magic; - uint64_t version; - uint64_t fileSize; - uint64_t resolveDBSize; - }; + static const uint64_t BufferAlignment; ////////////////////////////////////////// + + uint64_t m_SerVer; Mode m_Mode; @@ -761,7 +651,6 @@ class Serialiser int m_Indent; - bool m_HasResolver; Callstack::Stackwalk *m_pCallstack; Callstack::StackResolver *m_pResolver; Threading::ThreadHandle m_ResolverThread; @@ -779,11 +668,27 @@ class Serialiser // reading from file: - // where in the actual on-disk file does the data start (ie. after header and symbol DB) - uint64_t m_FileStartOffset; + struct Section + { + Section() : type(eSectionType_Unknown), flags(eSectionFlag_None), fileoffset(0), compressedReader(NULL) {} + string name; + SectionType type; + SectionFlags flags; + + uint64_t fileoffset; + uint64_t size; + vector data; // some sections can be loaded entirely into memory + CompressedFileIO *compressedReader; + }; + + // this lists all sections in file order + vector m_Sections; + + // this lists known sections, some may be NULL + Section *m_KnownSections[eSectionType_Num]; // where does our in-memory window point to in the data stream. ie. m_pBuffer[0] is - // m_ReadOffset into the disk stream + // m_ReadOffset into the frame capture section uint64_t m_ReadOffset; // how big is the current in-memory window @@ -821,7 +726,6 @@ class ScopedContext , m_DebugSer(debugser) #endif { - m_Alignment = 0; m_Name = string(n) + " = " + t; m_Ser->PushContext(m_Name.c_str(), m_Idx, smallChunk); @@ -839,7 +743,6 @@ class ScopedContext , m_DebugSer(debugser) #endif { - m_Alignment = 0; m_Name = n; m_Ser->PushContext(m_Name.c_str(), m_Idx, smallChunk); @@ -857,20 +760,14 @@ class ScopedContext End(); } - void SetAlignment(size_t align) - { - m_Alignment = align; - } - Chunk *Get(bool temporary = false) { End(); - return new Chunk(m_Ser, m_Idx, m_Alignment, temporary); + return new Chunk(m_Ser, m_Idx, temporary); } private: std::string m_Name; uint32_t m_Idx; - size_t m_Alignment; Serialiser *m_Ser; #ifdef DEBUG_TEXT_SERIALISER Serialiser *m_DebugSer;