fix: improve memory pressure from getlogs

This commit is contained in:
garethgeorge
2024-11-13 18:18:21 -08:00
parent b7abbf9603
commit 592e4cf9dd
3 changed files with 38 additions and 33 deletions

View File

@@ -504,54 +504,66 @@ func (s *BackrestHandler) GetLogs(ctx context.Context, req *connect.Request[v1.L
r.Close() r.Close()
}() }()
var bufferMu sync.Mutex
var buffer bytes.Buffer
var errChan = make(chan error, 1) var errChan = make(chan error, 1)
var sendChan = make(chan []byte, 1)
var buffer bytes.Buffer
var bufferMu sync.Mutex
go func() { go func() {
data := make([]byte, 4*1024) data := make([]byte, 4*1024)
for { for {
n, err := r.Read(data) n, err := r.Read(data)
if n == 0 { if n == 0 {
close(errChan)
break break
} else if err != nil && err != io.EOF { } else if err != nil && err != io.EOF {
errChan <- fmt.Errorf("failed to read log data: %w", err) errChan <- fmt.Errorf("failed to read log data: %w", err)
close(errChan) close(errChan)
return break
} }
bufferMu.Lock() bufferMu.Lock()
buffer.Write(data[:n]) buffer.Write(data[:n])
if buffer.Len() > 128*1024 {
sendChan <- bytes.Clone(buffer.Bytes())
buffer.Reset()
}
bufferMu.Unlock() bufferMu.Unlock()
} }
if buffer.Len() > 0 {
bufferMu.Lock()
sendChan <- bytes.Clone(buffer.Bytes())
buffer.Reset()
bufferMu.Unlock()
}
close(sendChan)
}() }()
ticker := time.NewTicker(100 * time.Millisecond) ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop() defer ticker.Stop()
flush := func() error {
bufferMu.Lock()
if buffer.Len() > 0 {
if err := resp.Send(&types.BytesValue{Value: buffer.Bytes()}); err != nil {
bufferMu.Unlock()
return fmt.Errorf("failed to send log data: %w", err)
}
buffer.Reset()
}
bufferMu.Unlock()
return nil
}
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
return flush() return nil
case err := <-errChan: case data, ok := <-sendChan:
_ = flush() if !ok {
return err return nil
case <-ticker.C: }
if err := flush(); err != nil { if err := resp.Send(&types.BytesValue{Value: data}); err != nil {
return err return err
} }
case err := <-errChan:
return err
case <-ticker.C:
bufferMu.Lock()
if buffer.Len() > 0 {
if err := resp.Send(&types.BytesValue{Value: bytes.Clone(buffer.Bytes())}); err != nil {
bufferMu.Unlock()
return err
}
buffer.Reset()
}
bufferMu.Unlock()
} }
} }

View File

@@ -30,14 +30,7 @@ import { shouldShowSettings } from "../state/configutil";
import { OpSelector } from "../../gen/ts/v1/service_pb"; import { OpSelector } from "../../gen/ts/v1/service_pb";
import { colorForStatus } from "../state/flowdisplayaggregator"; import { colorForStatus } from "../state/flowdisplayaggregator";
import { getStatusForSelector } from "../state/logstate"; import { getStatusForSelector } from "../state/logstate";
import { import { Route, Routes, useNavigate, useParams } from "react-router-dom";
createHashRouter,
Route,
RouterProvider,
Routes,
useNavigate,
useParams,
} from "react-router-dom";
import { MainContentAreaTemplate } from "./MainContentArea"; import { MainContentAreaTemplate } from "./MainContentArea";
const { Header, Sider } = Layout; const { Header, Sider } = Layout;

View File

@@ -202,12 +202,12 @@ const SummaryPanel = ({
</Typography.Text> </Typography.Text>
)} )}
{summary.backupsFailed30days && ( {summary.backupsFailed30days && (
<Typography.Text type="danger"> <Typography.Text type="danger" style={{ marginRight: "5px" }}>
{summary.backupsFailed30days + ""} failed {summary.backupsFailed30days + ""} failed
</Typography.Text> </Typography.Text>
)} )}
{summary.backupsWarningLast30days && ( {summary.backupsWarningLast30days && (
<Typography.Text type="warning"> <Typography.Text type="warning" style={{ marginRight: "5px" }}>
{summary.backupsWarningLast30days + ""} warning {summary.backupsWarningLast30days + ""} warning
</Typography.Text> </Typography.Text>
)} )}