From d71d275dc442df6d7413bed9110dbaaa54dbd624 Mon Sep 17 00:00:00 2001 From: baldurk Date: Wed, 2 Sep 2020 16:32:36 +0100 Subject: [PATCH] Don't re-read entire logfile every time, only read from last position --- qrenderdoc/Windows/LogView.cpp | 48 +++++++++++++------------ qrenderdoc/Windows/LogView.h | 2 +- renderdoc/api/replay/renderdoc_replay.h | 3 +- renderdoc/core/core.cpp | 2 +- renderdoc/os/os_specific.h | 7 ++-- renderdoc/os/posix/posix_stringio.cpp | 13 ++++--- renderdoc/os/win32/win32_stringio.cpp | 44 ++++++++++++++--------- renderdoc/replay/entry_points.cpp | 7 ++-- 8 files changed, 75 insertions(+), 51 deletions(-) diff --git a/qrenderdoc/Windows/LogView.cpp b/qrenderdoc/Windows/LogView.cpp index 64c238060..fcfc15abd 100644 --- a/qrenderdoc/Windows/LogView.cpp +++ b/qrenderdoc/Windows/LogView.cpp @@ -48,10 +48,11 @@ class LogItemModel : public QAbstractItemModel { public: LogItemModel(LogView *view) : QAbstractItemModel(view), m_Viewer(view) {} - void refresh() + void addRows(int numLines) { - emit beginResetModel(); - emit endResetModel(); + int count = rowCount(); + emit beginInsertRows(QModelIndex(), count - numLines, count - 1); + emit endInsertRows(); } QModelIndex index(int row, int column, const QModelIndex &parent = QModelIndex()) const override @@ -157,19 +158,23 @@ public: void refresh() { emit beginResetModel(); - - int numRows = sourceModel()->rowCount(); - - m_VisibleRows.clear(); - m_VisibleRows.reserve(numRows); - - for(int i = 0; i < numRows; i++) - if(isVisibleRow(i)) - m_VisibleRows.push_back(i); - emit endResetModel(); } + void addRows(int addedRows) + { + int numRows = sourceModel()->rowCount(); + emit beginInsertRows(QModelIndex(), numRows - addedRows, numRows - 1); + + m_VisibleRows.reserve(m_VisibleRows.count() + addedRows); + + for(int i = 0; i < addedRows; i++) + if(isVisibleRow(numRows - addedRows + i)) + m_VisibleRows.push_back(numRows - addedRows + i); + + emit endInsertRows(); + } + virtual QModelIndex mapFromSource(const QModelIndex &sourceIndex) const override { auto it = std::lower_bound(m_VisibleRows.begin(), m_VisibleRows.end(), sourceIndex.row()); @@ -188,7 +193,7 @@ public: if(proxyIndex.row() >= 0 && proxyIndex.row() < m_VisibleRows.count()) row = m_VisibleRows[proxyIndex.row()]; - return sourceModel()->index(row, proxyIndex.column(), proxyIndex.parent()); + return sourceModel()->index(row, proxyIndex.column()); } virtual int rowCount(const QModelIndex &parent = QModelIndex()) const override @@ -330,7 +335,6 @@ LogView::~LogView() m_Ctx.BuiltinWindowClosed(this); m_Messages.clear(); - m_ItemModel->refresh(); delete ui; } @@ -362,7 +366,7 @@ void LogView::on_save_clicked() } rdcstr contents; - RENDERDOC_GetLogFileContents(contents); + RENDERDOC_GetLogFileContents(0, contents); f->write(QByteArray(contents.c_str(), contents.count())); @@ -480,14 +484,14 @@ void LogView::pidFilter_changed(QStandardItem *item) void LogView::messages_refresh() { rdcstr contents; - RENDERDOC_GetLogFileContents(contents); + RENDERDOC_GetLogFileContents(prevOffset, contents); - if(prevOffset == contents.size()) + if(contents.empty()) return; // look at all new lines since the last one we saw - QStringList lines = QString(contents.substr(prevOffset)).split(QRegularExpression(lit("[\r\n]"))); - prevOffset = contents.size(); + QStringList lines = QString(contents).split(QRegularExpression(lit("[\r\n]"))); + prevOffset += contents.size(); QString r = lit("^" // start of the line @@ -538,8 +542,8 @@ void LogView::messages_refresh() if(!lines.isEmpty()) { - m_ItemModel->refresh(); - m_FilterModel->refresh(); + m_ItemModel->addRows(m_Messages.count() - prevCount); + m_FilterModel->addRows(m_Messages.count() - prevCount); } if(ui->followNew->isChecked()) diff --git a/qrenderdoc/Windows/LogView.h b/qrenderdoc/Windows/LogView.h index 22b4c9a78..a65f426bb 100644 --- a/qrenderdoc/Windows/LogView.h +++ b/qrenderdoc/Windows/LogView.h @@ -81,7 +81,7 @@ private: Ui::LogView *ui; ICaptureContext &m_Ctx; - size_t prevOffset = 0; + uint64_t prevOffset = 0; QVector m_Messages; diff --git a/renderdoc/api/replay/renderdoc_replay.h b/renderdoc/api/replay/renderdoc_replay.h index 49ab348bb..43b3fe04c 100644 --- a/renderdoc/api/replay/renderdoc_replay.h +++ b/renderdoc/api/replay/renderdoc_replay.h @@ -1980,7 +1980,8 @@ analysis program. extern "C" RENDERDOC_API const char *RENDERDOC_CC RENDERDOC_GetLogFile(); DOCUMENT("Internal function for fetching the contents of a log"); -extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_GetLogFileContents(rdcstr &logfile); +extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_GetLogFileContents(uint64_t offset, + rdcstr &logfile); DOCUMENT("Internal function for logging text simply."); extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_LogText(const char *text); diff --git a/renderdoc/core/core.cpp b/renderdoc/core/core.cpp index 110f27ddf..6dbd5fe64 100644 --- a/renderdoc/core/core.cpp +++ b/renderdoc/core/core.cpp @@ -1688,7 +1688,7 @@ void RenderDoc::FinishCaptureWriting(RDCFile *rdc, uint32_t frameNumber) if(Capture_Debug_SnapshotDiagnosticLog()) { - rdcstr logcontents = FileIO::logfile_readall(RDCGETLOGFILE()); + rdcstr logcontents = FileIO::logfile_readall(0, RDCGETLOGFILE()); SectionProperties props = {}; props.type = SectionType::EmbeddedLogfile; diff --git a/renderdoc/os/os_specific.h b/renderdoc/os/os_specific.h index 0638d9070..a639d7b24 100644 --- a/renderdoc/os/os_specific.h +++ b/renderdoc/os/os_specific.h @@ -325,9 +325,10 @@ LogFileHandle *logfile_open(const char *filename); void logfile_append(LogFileHandle *logHandle, const char *msg, size_t length); void logfile_close(LogFileHandle *logHandle, const char *deleteFilename); -// read the whole logfile into memory. This may race with processes writing, but it will read the -// whole of the file at some point. Useful since normal file reading may fail on the shared logfile -rdcstr logfile_readall(const char *filename); +// read the whole logfile into memory starting at a given offset. This may race with processes +// writing, but it will read the whole of the file at some point. Useful since normal file reading +// may fail on the shared logfile +rdcstr logfile_readall(uint64_t offset, const char *filename); // utility functions inline bool WriteAll(const rdcstr &filename, const void *buffer, size_t size) diff --git a/renderdoc/os/posix/posix_stringio.cpp b/renderdoc/os/posix/posix_stringio.cpp index 0e516c022..6441c211a 100644 --- a/renderdoc/os/posix/posix_stringio.cpp +++ b/renderdoc/os/posix/posix_stringio.cpp @@ -466,7 +466,7 @@ void ReleaseFDAfterFork() close(log); } -rdcstr logfile_readall(const char *filename) +rdcstr logfile_readall(uint64_t offset, const char *filename) { FILE *f = FileIO::fopen(filename, "r"); @@ -477,11 +477,16 @@ rdcstr logfile_readall(const char *filename) FileIO::fseek64(f, 0, SEEK_END); uint64_t size = FileIO::ftell64(f); - FileIO::fseek64(f, 0, SEEK_SET); - ret.resize((size_t)size); + if(size > offset) + { + FileIO::fseek64(f, offset, SEEK_SET); - FileIO::fread(&ret[0], 1, ret.size(), f); + ret.resize(size_t(size - offset)); + + size_t numRead = FileIO::fread(ret.data(), 1, ret.size(), f); + ret.resize(numRead); + } FileIO::fclose(f); diff --git a/renderdoc/os/win32/win32_stringio.cpp b/renderdoc/os/win32/win32_stringio.cpp index abad26e0d..77050f991 100644 --- a/renderdoc/os/win32/win32_stringio.cpp +++ b/renderdoc/os/win32/win32_stringio.cpp @@ -661,7 +661,7 @@ LogFileHandle *logfile_open(const char *filename) FILE_ATTRIBUTE_NORMAL, NULL); } -static rdcstr logfile_readall_fallback(const wchar_t *filename) +static rdcstr logfile_readall_fallback(uint64_t offset, const wchar_t *filename) { // if CreateFile/ReadFile failed, fall back and try regular stdio FILE *f = NULL; @@ -670,14 +670,15 @@ static rdcstr logfile_readall_fallback(const wchar_t *filename) { ::_fseeki64(f, 0, SEEK_END); uint64_t filesize = ::_ftelli64(f); - ::_fseeki64(f, 0, SEEK_SET); - if(filesize > 10) + if(filesize > 10 && filesize > offset) { - rdcstr ret; - ret.resize((size_t)filesize); + ::_fseeki64(f, offset, SEEK_SET); - size_t numRead = ::fread(&ret[0], 1, (size_t)filesize, f); + rdcstr ret; + ret.resize(size_t(filesize - offset)); + + size_t numRead = ::fread(&ret[0], 1, ret.size(), f); ret.resize(numRead); ::fclose(f); @@ -691,11 +692,8 @@ static rdcstr logfile_readall_fallback(const wchar_t *filename) return ""; } -rdcstr logfile_readall(const char *filename) +rdcstr logfile_readall(uint64_t offset, const char *filename) { - if(!exists(filename)) - return StringFormat::Fmt("Logfile '%s' doesn't exist", filename); - rdcwstr wfn = StringFormat::UTF82Wide(filename); HANDLE h = CreateFileW(wfn.c_str(), FILE_READ_DATA, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); @@ -706,27 +704,41 @@ rdcstr logfile_readall(const char *filename) { DWORD err = GetLastError(); - ret = logfile_readall_fallback(wfn.c_str()); + if(err == ERROR_FILE_NOT_FOUND) + return StringFormat::Fmt("Logfile '%s' doesn't exist", filename); + + ret = logfile_readall_fallback(offset, wfn.c_str()); ret += StringFormat::Fmt("\n\nCouldn't open logfile, CreateFile() threw %u\n\n", err); } else { - DWORD len = ::GetFileSize(h, NULL); + DWORD highlen = 0; + DWORD len = ::GetFileSize(h, &highlen); if(len == INVALID_FILE_SIZE) { DWORD err = GetLastError(); - ret = logfile_readall_fallback(wfn.c_str()); + ret = logfile_readall_fallback(offset, wfn.c_str()); ret += StringFormat::Fmt("\n\nFailed to read logfile, GetFileSize() threw %u", err); } else { - ret.resize(len); + uint64_t length = uint64_t(highlen) << 32 | len; - DWORD dummy = len; + if(offset < length) + { + LARGE_INTEGER offs; + offs.LowPart = (offset & 0xFFFFFFFFU); + offs.HighPart = ((offset >> 32) & 0xFFFFFFFFU); + ::SetFilePointerEx(h, offs, NULL, FILE_BEGIN); - ReadFile(h, &ret[0], len, &dummy, NULL); + ret.resize(size_t(length - offset)); + + DWORD dummy = ret.count(); + + ReadFile(h, ret.data(), ret.count(), &dummy, NULL); + } } CloseHandle(h); diff --git a/renderdoc/replay/entry_points.cpp b/renderdoc/replay/entry_points.cpp index 95708ed64..ed8097111 100644 --- a/renderdoc/replay/entry_points.cpp +++ b/renderdoc/replay/entry_points.cpp @@ -278,9 +278,10 @@ extern "C" RENDERDOC_API const char *RENDERDOC_CC RENDERDOC_GetLogFile() return RDCGETLOGFILE(); } -extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_GetLogFileContents(rdcstr &logfile) +extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_GetLogFileContents(uint64_t offset, + rdcstr &logfile) { - logfile = FileIO::logfile_readall(RDCGETLOGFILE()); + logfile = FileIO::logfile_readall(offset, RDCGETLOGFILE()); } extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_InitialiseReplay(GlobalEnvironment env, @@ -335,7 +336,7 @@ extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_CreateBugReport(const char if(logfile && logfile[0]) { - rdcstr contents = FileIO::logfile_readall(logfile); + rdcstr contents = FileIO::logfile_readall(0, logfile); mz_zip_writer_add_mem(&zip, "error.log", contents.data(), contents.length(), MZ_BEST_COMPRESSION); }