built-in agent

fix(package-lock.json): fix for nixpkgs

beszel/internal/agent/agent.go

@@ -13,6 +13,8 @@ import (
 
 type Agent struct {
 	debug            bool                       // true if LOG_LEVEL is set to debug
+	zfs              bool                       // true if system has arcstats
+	memCalc          string                     // Memory calculation formula
 	fsNames          []string                   // List of filesystem device names being monitored
 	fsStats          map[string]*system.FsStats // Keeps track of disk stats for each filesystem
 	netInterfaces    map[string]struct{}        // Stores all valid network interfaces
@@ -26,6 +28,7 @@ type Agent struct {
 func NewAgent() *Agent {
 	return &Agent{
 		sensorsContext: context.Background(),
+		memCalc:        os.Getenv("MEM_CALC"),
 	}
 }
 
@@ -67,13 +70,15 @@ func (a *Agent) Run(pubKey []byte, addr string) {
 
 	// if debugging, print stats
 	if a.debug {
-		slog.Debug("Stats", "data", a.gatherStats())
+		slog.Debug("Stats", "data", a.GatherStats())
 	}
 
-	a.startServer(pubKey, addr)
+	if pubKey != nil {
+		a.startServer(pubKey, addr)
+	}
 }
 
-func (a *Agent) gatherStats() system.CombinedData {
+func (a *Agent) GatherStats() system.CombinedData {
 	systemData := system.CombinedData{
 		Stats: a.getSystemStats(),
 		Info:  a.systemInfo,

beszel/internal/agent/server.go

@@ -24,7 +24,7 @@ func (a *Agent) startServer(pubKey []byte, addr string) {
 }
 
 func (a *Agent) handleSession(s sshServer.Session) {
-	stats := a.gatherStats()
+	stats := a.GatherStats()
 	slog.Debug("Sending stats", "data", stats)
 	if err := json.NewEncoder(s).Encode(stats); err != nil {
 		slog.Error("Error encoding stats", "err", err)

beszel/internal/agent/system.go

@@ -3,9 +3,12 @@ package agent
 import (
 	"beszel"
 	"beszel/internal/entities/system"
+	"bufio"
+	"fmt"
 	"log/slog"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/shirou/gopsutil/v4/cpu"
@@ -36,6 +39,13 @@ func (a *Agent) initializeSystemInfo() {
 			a.systemInfo.Threads = threads
 		}
 	}
+
+	// zfs
+	if _, err := getARCSize(); err == nil {
+		a.zfs = true
+	} else {
+		slog.Debug("Not monitoring ZFS ARC", "err", err)
+	}
 }
 
 // Returns current info, stats about the host system
@@ -52,12 +62,30 @@ func (a *Agent) getSystemStats() system.Stats {
 
 	// memory
 	if v, err := mem.VirtualMemory(); err == nil {
-		systemStats.Mem = bytesToGigabytes(v.Total)
-		systemStats.MemUsed = bytesToGigabytes(v.Used)
-		systemStats.MemBuffCache = bytesToGigabytes(v.Total - v.Free - v.Used)
-		systemStats.MemPct = twoDecimals(v.UsedPercent)
+		// swap
 		systemStats.Swap = bytesToGigabytes(v.SwapTotal)
-		systemStats.SwapUsed = bytesToGigabytes(v.SwapTotal - v.SwapFree)
+		systemStats.SwapUsed = bytesToGigabytes(v.SwapTotal - v.SwapFree - v.SwapCached)
+		// cache + buffers value for default mem calculation
+		cacheBuff := v.Total - v.Free - v.Used
+		// htop memory calculation overrides
+		if a.memCalc == "htop" {
+			// note: gopsutil automatically adds SReclaimable to v.Cached
+			cacheBuff = v.Cached + v.Buffers - v.Shared
+			v.Used = v.Total - (v.Free + cacheBuff)
+			v.UsedPercent = float64(v.Used) / float64(v.Total) * 100.0
+		}
+		// subtract ZFS ARC size from used memory and add as its own category
+		if a.zfs {
+			if arcSize, _ := getARCSize(); arcSize > 0 && arcSize < v.Used {
+				v.Used = v.Used - arcSize
+				v.UsedPercent = float64(v.Used) / float64(v.Total) * 100.0
+				systemStats.MemZfsArc = bytesToGigabytes(arcSize)
+			}
+		}
+		systemStats.Mem = bytesToGigabytes(v.Total)
+		systemStats.MemBuffCache = bytesToGigabytes(cacheBuff)
+		systemStats.MemUsed = bytesToGigabytes(v.Used)
+		systemStats.MemPct = twoDecimals(v.UsedPercent)
 	}
 
 	// disk usage
@@ -154,7 +182,7 @@ func (a *Agent) getSystemStats() system.Stats {
 		systemStats.Temperatures = make(map[string]float64, len(temps))
 		for i, sensor := range temps {
 			// skip if temperature is 0
-			if sensor.Temperature == 0 {
+			if sensor.Temperature <= 0 || sensor.Temperature >= 200 {
 				continue
 			}
 			if _, ok := systemStats.Temperatures[sensor.SensorKey]; ok {
@@ -183,3 +211,29 @@ func (a *Agent) getSystemStats() system.Stats {
 
 	return systemStats
 }
+
+// Returns the size of the ZFS ARC memory cache in bytes
+func getARCSize() (uint64, error) {
+	file, err := os.Open("/proc/spl/kstat/zfs/arcstats")
+	if err != nil {
+		return 0, err
+	}
+	defer file.Close()
+
+	// Scan the lines
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "size") {
+			// Example line: size 4 15032385536
+			fields := strings.Fields(line)
+			if len(fields) < 3 {
+				return 0, err
+			}
+			// Return the size as uint64
+			return strconv.ParseUint(fields[2], 10, 64)
+		}
+	}
+
+	return 0, fmt.Errorf("failed to parse size field")
+}

beszel/internal/entities/system/system.go

@@ -11,6 +11,7 @@ type Stats struct {
 	MemUsed      float64             `json:"mu"`
 	MemPct       float64             `json:"mp"`
 	MemBuffCache float64             `json:"mb"`
+	MemZfsArc    float64             `json:"mz,omitempty"` // ZFS ARC memory
 	Swap         float64             `json:"s,omitempty"`
 	SwapUsed     float64             `json:"su,omitempty"`
 	DiskTotal    float64             `json:"d"`

beszel/internal/hub/hub.go

@@ -3,6 +3,7 @@ package hub
 
 import (
 	"beszel"
+	"beszel/internal/agent"
 	"beszel/internal/alerts"
 	"beszel/internal/entities/system"
 	"beszel/internal/records"
@@ -42,6 +43,7 @@ type Hub struct {
 	am                *alerts.AlertManager
 	um                *users.UserManager
 	rm                *records.RecordManager
+	hubAgent          *agent.Agent
 }
 
 func NewHub(app *pocketbase.PocketBase) *Hub {
@@ -56,10 +58,6 @@ func NewHub(app *pocketbase.PocketBase) *Hub {
 }
 
 func (h *Hub) Run() {
-	// rm := records.NewRecordManager(h.app)
-	// am := alerts.NewAlertManager(h.app)
-	// um := users.NewUserManager(h.app)
-
 	// loosely check if it was executed using "go run"
 	isGoRun := strings.HasPrefix(os.Args[0], os.TempDir())
 
@@ -73,25 +71,22 @@ func (h *Hub) Run() {
 	// initial setup
 	h.app.OnBeforeServe().Add(func(e *core.ServeEvent) error {
 		// create ssh client config
-		err := h.createSSHClientConfig()
-		if err != nil {
+		if err := h.createSSHClientConfig(); err != nil {
 			log.Fatal(err)
 		}
 		// set auth settings
-		usersCollection, err := h.app.Dao().FindCollectionByNameOrId("users")
-		if err != nil {
-			return err
-		}
-		usersAuthOptions := usersCollection.AuthOptions()
-		usersAuthOptions.AllowUsernameAuth = false
-		if os.Getenv("DISABLE_PASSWORD_AUTH") == "true" {
-			usersAuthOptions.AllowEmailAuth = false
-		} else {
-			usersAuthOptions.AllowEmailAuth = true
-		}
-		usersCollection.SetOptions(usersAuthOptions)
-		if err := h.app.Dao().SaveCollection(usersCollection); err != nil {
-			return err
+		if usersCollection, err := h.app.Dao().FindCollectionByNameOrId("users"); err == nil {
+			usersAuthOptions := usersCollection.AuthOptions()
+			usersAuthOptions.AllowUsernameAuth = false
+			if os.Getenv("DISABLE_PASSWORD_AUTH") == "true" {
+				usersAuthOptions.AllowEmailAuth = false
+			} else {
+				usersAuthOptions.AllowEmailAuth = true
+			}
+			usersCollection.SetOptions(usersAuthOptions)
+			if err := h.app.Dao().SaveCollection(usersCollection); err != nil {
+				return err
+			}
 		}
 		return nil
 	})
@@ -159,6 +154,16 @@ func (h *Hub) Run() {
 	// system creation defaults
 	h.app.OnModelBeforeCreate("systems").Add(func(e *core.ModelEvent) error {
 		record := e.Model.(*models.Record)
+		if record.GetString("host") == "hubsys" {
+			// todo: check for hubsys existance and return error if exists (or make sure user is admin)
+			if record.GetString("name") == "x" {
+				hostname, _ := os.Hostname()
+				if hostname == "" {
+					hostname = "localhost"
+				}
+				record.Set("name", hostname)
+			}
+		}
 		record.Set("info", system.Info{})
 		record.Set("status", "pending")
 		return nil
@@ -246,6 +251,26 @@ func (h *Hub) updateSystems() {
 }
 
 func (h *Hub) updateSystem(record *models.Record) {
+	switch record.GetString("host") {
+	case "hubsys":
+		h.updateHubSystem(record)
+	default:
+		h.updateRemoteSystem(record)
+	}
+}
+
+// Update hub system stats with built-in agent
+func (h *Hub) updateHubSystem(record *models.Record) {
+	if h.hubAgent == nil {
+		h.hubAgent = agent.NewAgent()
+		h.hubAgent.Run(nil, "")
+	}
+	systemData := h.hubAgent.GatherStats()
+	h.saveSystemStats(record, &systemData)
+}
+
+// Connect to remote system and update system stats
+func (h *Hub) updateRemoteSystem(record *models.Record) {
 	var client *ssh.Client
 	var err error
 
@@ -273,7 +298,7 @@ func (h *Hub) updateSystem(record *models.Record) {
 			// if previous connection was closed, try again
 			h.app.Logger().Error("Existing SSH connection closed. Retrying...", "host", record.GetString("host"), "port", record.GetString("port"))
 			h.deleteSystemConnection(record)
-			h.updateSystem(record)
+			h.updateRemoteSystem(record)
 			return
 		}
 		h.app.Logger().Error("Failed to get system stats: ", "err", err.Error())
@@ -281,6 +306,11 @@ func (h *Hub) updateSystem(record *models.Record) {
 		return
 	}
 	// update system record
+	h.saveSystemStats(record, &systemData)
+}
+
+// Update system record with provided system.CombinedData
+func (h *Hub) saveSystemStats(record *models.Record, systemData *system.CombinedData) {
 	record.Set("status", "up")
 	record.Set("info", systemData.Info)
 	if err := h.app.Dao().SaveRecord(record); err != nil {
@@ -320,14 +350,20 @@ func (h *Hub) updateSystemStatus(record *models.Record, status string) {
 	}
 }
 
+// Deletes the SSH connection (remote) or built-in agent reference
 func (h *Hub) deleteSystemConnection(record *models.Record) {
-	if _, ok := h.systemConnections[record.Id]; ok {
-		if h.systemConnections[record.Id] != nil {
-			h.systemConnections[record.Id].Close()
+	switch record.GetString("host") {
+	case "hubsys":
+		h.hubAgent = nil
+	default:
+		if _, ok := h.systemConnections[record.Id]; ok {
+			if h.systemConnections[record.Id] != nil {
+				h.systemConnections[record.Id].Close()
+			}
+			h.connectionLock.Lock()
+			defer h.connectionLock.Unlock()
+			delete(h.systemConnections, record.Id)
 		}
-		h.connectionLock.Lock()
-		defer h.connectionLock.Unlock()
-		delete(h.systemConnections, record.Id)
 	}
 }
 

beszel/internal/records/records.go

@@ -161,6 +161,7 @@ func (rm *RecordManager) AverageSystemStats(records []*models.Record) system.Sta
 		sum.MemUsed += stats.MemUsed
 		sum.MemPct += stats.MemPct
 		sum.MemBuffCache += stats.MemBuffCache
+		sum.MemZfsArc += stats.MemZfsArc
 		sum.Swap += stats.Swap
 		sum.SwapUsed += stats.SwapUsed
 		sum.DiskTotal += stats.DiskTotal
@@ -200,6 +201,7 @@ func (rm *RecordManager) AverageSystemStats(records []*models.Record) system.Sta
 		MemUsed:      twoDecimals(sum.MemUsed / count),
 		MemPct:       twoDecimals(sum.MemPct / count),
 		MemBuffCache: twoDecimals(sum.MemBuffCache / count),
+		MemZfsArc:    twoDecimals(sum.MemZfsArc / count),
 		Swap:         twoDecimals(sum.Swap / count),
 		SwapUsed:     twoDecimals(sum.SwapUsed / count),
 		DiskTotal:    twoDecimals(sum.DiskTotal / count),

beszel/site/src/components/charts/mem-chart.tsx

@@ -5,12 +5,11 @@ import {
 	useYAxisWidth,
 	chartTimeData,
 	cn,
-	formatShortDate,
 	toFixedFloat,
 	twoDecimalString,
+	formatShortDate,
 } from '@/lib/utils'
 import { useMemo } from 'react'
-// import Spinner from '../spinner'
 import { useStore } from '@nanostores/react'
 import { $chartTime } from '@/lib/stores'
 import { SystemStatsRecord } from '@/types'
@@ -79,7 +78,7 @@ export default function MemChart({
 						content={
 							<ChartTooltipContent
 								// @ts-ignore
-								itemSorter={(a, b) => a.name.localeCompare(b.name)}
+								itemSorter={(a, b) => a.order - b.order}
 								labelFormatter={(_, data) => formatShortDate(data[0].payload.created)}
 								contentFormatter={(item) => twoDecimalString(item.value) + ' GB'}
 								indicator="line"
@@ -87,8 +86,9 @@ export default function MemChart({
 						}
 					/>
 					<Area
-						dataKey="stats.mu"
 						name="Used"
+						order={3}
+						dataKey="stats.mu"
 						type="monotoneX"
 						fill="hsl(var(--chart-2))"
 						fillOpacity={0.4}
@@ -96,9 +96,23 @@ export default function MemChart({
 						stackId="1"
 						isAnimationActive={false}
 					/>
+					{systemData.at(-1)?.stats.mz && (
+						<Area
+							name="ZFS ARC"
+							order={2}
+							dataKey="stats.mz"
+							type="monotoneX"
+							fill="hsla(175 60% 45% / 0.8)"
+							fillOpacity={0.5}
+							stroke="hsla(175 60% 45% / 0.8)"
+							stackId="1"
+							isAnimationActive={false}
+						/>
+					)}
 					<Area
-						dataKey="stats.mb"
 						name="Cache / Buffers"
+						order={1}
+						dataKey="stats.mb"
 						type="monotoneX"
 						fill="hsla(160 60% 45% / 0.5)"
 						fillOpacity={0.4}

beszel/site/src/components/login/auth-form.tsx

@@ -94,13 +94,15 @@ export function UserAuthForm({
 						setErrors({ passwordConfirm: msg })
 						return
 					}
+					// create admin user
 					await pb.admins.create({
 						email,
 						password,
 						passwordConfirm: password,
 					})
 					await pb.admins.authWithPassword(email, password)
-					await pb.collection('users').create({
+					// create regular user
+					const user = await pb.collection('users').create({
 						username,
 						email,
 						password,
@@ -108,6 +110,13 @@ export function UserAuthForm({
 						role: 'admin',
 						verified: true,
 					})
+					// create hubsys
+					await pb.collection('systems').create({
+						name: 'x',
+						port: 'x',
+						host: 'hubsys',
+						users: user.id,
+					})
 					await pb.collection('users').authWithPassword(email, password)
 				} else {
 					await pb.collection('users').authWithPassword(email, password)

beszel/site/src/components/routes/system.tsx

@@ -197,7 +197,7 @@ export default function SystemDetail({ name }: { name: string }) {
 			uptime = `${Math.trunc(system.info?.u / 86400)} days`
 		}
 		return [
-			{ value: system.host, Icon: GlobeIcon },
+			{ value: system.host, Icon: GlobeIcon, hide: system.host === 'hubsys' },
 			{
 				value: system.info.h,
 				Icon: MonitorIcon,

beszel/site/src/types.d.ts

@@ -43,6 +43,8 @@ export interface SystemStats {
 	mp: number
 	/** memory buffer + cache (gb) */
 	mb: number
+	/** zfs arc memory (gb) */
+	mz?: number
 	/** swap space (gb) */
 	s: number
 	/** swap used (gb) */

beszel/version.go

@@ -1,6 +1,6 @@
 package beszel
 
 const (
-	Version = "0.5.1"
+	Version = "0.5.2"
 	AppName = "beszel"
 )

readme.md

@@ -105,20 +105,21 @@ Use `./beszel update` and `./beszel-agent update` to update to the latest versio
 
 ### Agent
 
-| Name                | Default | Description                                                                              |
-| ------------------- | ------- | ---------------------------------------------------------------------------------------- |
-| `DOCKER_HOST`       | unset   | Overrides the docker host (docker.sock) if using a proxy.[^socket]                       |
+| Name                | Default | Description                                                                                                               |
+| ------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------- |
+| `DOCKER_HOST`       | unset   | Overrides the docker host (docker.sock) if using a proxy.[^socket]                                                        |
 | `EXTRA_FILESYSTEMS` | unset   | See [Monitoring additional disks, partitions, or remote mounts](#monitoring-additional-disks-partitions-or-remote-mounts) |
-| `FILESYSTEM`        | unset   | Device, partition, or mount point to use for root disk stats.                            |
-| `KEY`               | unset   | Public SSH key to use for authentication. Provided in hub.                               |
-| `LOG_LEVEL`         | info    | Logging level. Valid values: "debug", "info", "warn", "error".                           |
-| `NICS`              | unset   | Whitelist of network interfaces to monitor for bandwidth chart.                          |
-| `PORT`              | 45876   | Port or address:port to listen on.                                                       |
-| `SENSORS`           | unset   | Whitelist of temperature sensors to monitor.                                             |
-
-<!-- | `SYS_SENSORS` | unset | Overrides the sys location for sensors. | -->
+| `FILESYSTEM`        | unset   | Device, partition, or mount point to use for root disk stats.                                                             |
+| `KEY`               | unset   | Public SSH key to use for authentication. Provided in hub.                                                                |
+| `LOG_LEVEL`         | info    | Logging level. Valid values: "debug", "info", "warn", "error".                                                            |
+| `MEM_CALC`          | unset   | Overrides the default memory calculation.[^memcalc]                                                                       |
+| `NICS`              | unset   | Whitelist of network interfaces to monitor for bandwidth chart.                                                           |
+| `PORT`              | 45876   | Port or address:port to listen on.                                                                                        |
+| `SENSORS`           | unset   | Whitelist of temperature sensors to monitor.                                                                              |
+| `SYS_SENSORS`       | unset   | Overrides sys path for sensors. See [#160](https://github.com/henrygd/beszel/discussions/160).                            |
 
 [^socket]: Beszel only needs access to read container information. For [linuxserver/docker-socket-proxy](https://github.com/linuxserver/docker-socket-proxy) you would set `CONTAINERS=1`.
+[^memcalc]: The default value for used memory is based on gopsutil's [Used](https://pkg.go.dev/github.com/shirou/gopsutil/v4@v4.24.6/mem#VirtualMemoryStat) calculation, which should align fairly closely with `free`. Set `MEM_CALC` to `htop` to align with htop's calculation.
 
 ## OAuth / OIDC Setup