0.0.1

subtract mem cache from container stats

agent/main.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"sync"
 	"time"
 
 	sshServer "github.com/gliderlabs/ssh"
@@ -21,11 +22,20 @@ import (
 	psutilNet "github.com/shirou/gopsutil/v4/net"
 )
 
-var Version = "0.0.1-alpha.7"
+var Version = "0.0.1"
 
 var containerCpuMap = make(map[string][2]uint64)
+var containerCpuMutex = &sync.Mutex{}
 
-// var containerCpuMutex = &sync.Mutex{}
+var sem = make(chan struct{}, 15)
+
+func acquireSemaphore() {
+	sem <- struct{}{}
+}
+
+func releaseSemaphore() {
+	<-sem
+}
 
 var diskIoStats = DiskIoStats{
 	Read:       0,
@@ -48,54 +58,15 @@ var client = &http.Client{
 		Dial: func(proto, addr string) (net.Conn, error) {
 			return net.Dial("unix", "/var/run/docker.sock")
 		},
-		ForceAttemptHTTP2:  false,
-		IdleConnTimeout:    90 * time.Second,
-		DisableCompression: true,
-		MaxIdleConns:       10,
-		DisableKeepAlives:  false,
+		ForceAttemptHTTP2:   false,
+		IdleConnTimeout:     90 * time.Second,
+		DisableCompression:  true,
+		MaxIdleConnsPerHost: 50,
+		DisableKeepAlives:   false,
 	},
 }
 
-type SystemData struct {
-	Stats      SystemStats      `json:"stats"`
-	Info       SystemInfo       `json:"info"`
-	Containers []ContainerStats `json:"container"`
-}
-
-type SystemInfo struct {
-	Cores    int    `json:"c"`
-	Threads  int    `json:"t"`
-	CpuModel string `json:"m"`
-	// Os       string  `json:"o"`
-	Uptime  uint64  `json:"u"`
-	Cpu     float64 `json:"cpu"`
-	MemPct  float64 `json:"mp"`
-	DiskPct float64 `json:"dp"`
-}
-
-type SystemStats struct {
-	Cpu          float64 `json:"cpu"`
-	Mem          float64 `json:"m"`
-	MemUsed      float64 `json:"mu"`
-	MemPct       float64 `json:"mp"`
-	MemBuffCache float64 `json:"mb"`
-	Disk         float64 `json:"d"`
-	DiskUsed     float64 `json:"du"`
-	DiskPct      float64 `json:"dp"`
-	DiskRead     float64 `json:"dr"`
-	DiskWrite    float64 `json:"dw"`
-	NetworkSent  float64 `json:"ns"`
-	NetworkRecv  float64 `json:"nr"`
-}
-
-type ContainerStats struct {
-	Name string  `json:"n"`
-	Cpu  float64 `json:"c"`
-	Mem  float64 `json:"m"`
-	// MemPct float64 `json:"mp"`
-}
-
-func getSystemStats() (SystemInfo, SystemStats) {
+func getSystemStats() (*SystemInfo, *SystemStats) {
 	c, _ := cpu.Percent(0, false)
 	v, _ := mem.VirtualMemory()
 	d, _ := disk.Usage("/")
@@ -104,7 +75,7 @@ func getSystemStats() (SystemInfo, SystemStats) {
 	memPct := twoDecimals(v.UsedPercent)
 	diskPct := twoDecimals(d.UsedPercent)
 
-	systemStats := SystemStats{
+	systemStats := &SystemStats{
 		Cpu:          cpuPct,
 		Mem:          bytesToGigabytes(v.Total),
 		MemUsed:      bytesToGigabytes(v.Used),
@@ -115,7 +86,7 @@ func getSystemStats() (SystemInfo, SystemStats) {
 		DiskPct:      diskPct,
 	}
 
-	systemInfo := SystemInfo{
+	systemInfo := &SystemInfo{
 		Cpu:     cpuPct,
 		MemPct:  memPct,
 		DiskPct: diskPct,
@@ -181,51 +152,63 @@ func getSystemStats() (SystemInfo, SystemStats) {
 
 }
 
-func getDockerStats() ([]ContainerStats, error) {
+func getDockerStats() ([]*ContainerStats, error) {
 	resp, err := client.Get("http://localhost/containers/json")
 	if err != nil {
-		return []ContainerStats{}, err
+		return []*ContainerStats{}, err
 	}
 	defer resp.Body.Close()
 
-	var containers []Container
+	var containers []*Container
 	if err := json.NewDecoder(resp.Body).Decode(&containers); err != nil {
 		panic(err)
 	}
 
-	var containerStats []ContainerStats
+	containerStats := make([]*ContainerStats, 0, len(containers))
+
+	// store valid ids to clean up old container ids from map
+	validIds := make(map[string]struct{}, len(containers))
+
+	var wg sync.WaitGroup
 
 	for _, ctr := range containers {
-		cstats, err := getContainerStats(ctr)
-		if err != nil {
-			// retry once
-			cstats, err = getContainerStats(ctr)
+		ctr.IdShort = ctr.ID[:12]
+		validIds[ctr.IdShort] = struct{}{}
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			cstats, err := getContainerStats(ctr)
 			if err != nil {
-				log.Printf("Error getting container stats: %+v\n", err)
-				continue
+				// retry once
+				cstats, err = getContainerStats(ctr)
+				if err != nil {
+					log.Printf("Error getting container stats: %+v\n", err)
+					return
+				}
 			}
-		}
-		containerStats = append(containerStats, cstats)
+			containerStats = append(containerStats, cstats)
+		}()
 	}
 
-	// clean up old containers from map
-	validNames := make(map[string]struct{}, len(containers))
-	for _, ctr := range containers {
-		validNames[ctr.Names[0][1:]] = struct{}{}
-	}
-	for name := range containerCpuMap {
-		if _, exists := validNames[name]; !exists {
-			delete(containerCpuMap, name)
+	wg.Wait()
+
+	for id := range containerCpuMap {
+		if _, exists := validIds[id]; !exists {
+			// log.Printf("Removing container cpu map entry: %+v\n", id)
+			delete(containerCpuMap, id)
 		}
 	}
 
 	return containerStats, nil
 }
 
-func getContainerStats(ctr Container) (ContainerStats, error) {
-	resp, err := client.Get("http://localhost/containers/" + ctr.ID + "/stats?stream=0&one-shot=1")
+func getContainerStats(ctr *Container) (*ContainerStats, error) {
+	// use semaphore to limit concurrency
+	acquireSemaphore()
+	defer releaseSemaphore()
+	resp, err := client.Get("http://localhost/containers/" + ctr.IdShort + "/stats?stream=0&one-shot=1")
 	if err != nil {
-		return ContainerStats{}, err
+		return &ContainerStats{}, err
 	}
 	defer resp.Body.Close()
 
@@ -236,26 +219,30 @@ func getContainerStats(ctr Container) (ContainerStats, error) {
 
 	name := ctr.Names[0][1:]
 
-	// memory
-	usedMemory := statsJson.MemoryStats.Usage - statsJson.MemoryStats.Cache
+	// memory (https://docs.docker.com/reference/cli/docker/container/stats/)
+	memCache := statsJson.MemoryStats.Stats["inactive_file"]
+	if memCache == 0 {
+		memCache = statsJson.MemoryStats.Stats["cache"]
+	}
+	usedMemory := statsJson.MemoryStats.Usage - memCache
 	// pctMemory := float64(usedMemory) / float64(statsJson.MemoryStats.Limit) * 100
 
 	// cpu
 	// add default values to containerCpu if it doesn't exist
-	// containerCpuMutex.Lock()
-	// defer containerCpuMutex.Unlock()
-	if _, ok := containerCpuMap[name]; !ok {
-		containerCpuMap[name] = [2]uint64{0, 0}
+	containerCpuMutex.Lock()
+	defer containerCpuMutex.Unlock()
+	if _, ok := containerCpuMap[ctr.IdShort]; !ok {
+		containerCpuMap[ctr.IdShort] = [2]uint64{0, 0}
 	}
-	cpuDelta := statsJson.CPUStats.CPUUsage.TotalUsage - containerCpuMap[name][0]
-	systemDelta := statsJson.CPUStats.SystemUsage - containerCpuMap[name][1]
+	cpuDelta := statsJson.CPUStats.CPUUsage.TotalUsage - containerCpuMap[ctr.IdShort][0]
+	systemDelta := statsJson.CPUStats.SystemUsage - containerCpuMap[ctr.IdShort][1]
 	cpuPct := float64(cpuDelta) / float64(systemDelta) * 100
 	if cpuPct > 100 {
-		return ContainerStats{}, fmt.Errorf("%s cpu pct greater than 100: %+v", name, cpuPct)
+		return &ContainerStats{}, fmt.Errorf("%s cpu pct greater than 100: %+v", name, cpuPct)
 	}
-	containerCpuMap[name] = [2]uint64{statsJson.CPUStats.CPUUsage.TotalUsage, statsJson.CPUStats.SystemUsage}
+	containerCpuMap[ctr.IdShort] = [2]uint64{statsJson.CPUStats.CPUUsage.TotalUsage, statsJson.CPUStats.SystemUsage}
 
-	cStats := ContainerStats{
+	cStats := &ContainerStats{
 		Name: name,
 		Cpu:  twoDecimals(cpuPct),
 		Mem:  bytesToMegabytes(float64(usedMemory)),
@@ -264,12 +251,12 @@ func getContainerStats(ctr Container) (ContainerStats, error) {
 	return cStats, nil
 }
 
-func gatherStats() SystemData {
+func gatherStats() *SystemData {
 	systemInfo, systemStats := getSystemStats()
-	stats := SystemData{
+	stats := &SystemData{
 		Stats:      systemStats,
 		Info:       systemInfo,
-		Containers: []ContainerStats{},
+		Containers: []*ContainerStats{},
 	}
 	containerStats, err := getDockerStats()
 	if err == nil {

agent/types.go

@@ -2,8 +2,48 @@ package main
 
 import "time"
 
+type SystemData struct {
+	Stats      *SystemStats      `json:"stats"`
+	Info       *SystemInfo       `json:"info"`
+	Containers []*ContainerStats `json:"container"`
+}
+
+type SystemInfo struct {
+	Cores    int    `json:"c"`
+	Threads  int    `json:"t"`
+	CpuModel string `json:"m"`
+	// Os       string  `json:"o"`
+	Uptime  uint64  `json:"u"`
+	Cpu     float64 `json:"cpu"`
+	MemPct  float64 `json:"mp"`
+	DiskPct float64 `json:"dp"`
+}
+
+type SystemStats struct {
+	Cpu          float64 `json:"cpu"`
+	Mem          float64 `json:"m"`
+	MemUsed      float64 `json:"mu"`
+	MemPct       float64 `json:"mp"`
+	MemBuffCache float64 `json:"mb"`
+	Disk         float64 `json:"d"`
+	DiskUsed     float64 `json:"du"`
+	DiskPct      float64 `json:"dp"`
+	DiskRead     float64 `json:"dr"`
+	DiskWrite    float64 `json:"dw"`
+	NetworkSent  float64 `json:"ns"`
+	NetworkRecv  float64 `json:"nr"`
+}
+
+type ContainerStats struct {
+	Name string  `json:"n"`
+	Cpu  float64 `json:"c"`
+	Mem  float64 `json:"m"`
+	// MemPct float64 `json:"mp"`
+}
+
 type Container struct {
 	ID      string `json:"Id"`
+	IdShort string
 	Names   []string
 	Image   string
 	ImageID string

hub/main.go

@@ -30,7 +30,7 @@ import (
 	"golang.org/x/crypto/ssh"
 )
 
-var Version = "0.0.1-alpha.7"
+var Version = "0.0.1"
 
 var app *pocketbase.PocketBase
 var serverConnections = make(map[string]*Server)
@@ -217,14 +217,14 @@ func updateSystems() {
 		0,                    // offset
 	)
 	// log.Println("records", len(records))
-	if err != nil {
-		app.Logger().Error("Failed to query systems: ", "err", err.Error())
+	if err != nil || len(records) == 0 {
+		// app.Logger().Error("Failed to query systems")
 		return
 	}
-	fiftyFiveSecondsAgo := time.Now().UTC().Add(-55 * time.Second)
+	fiftySecondsAgo := time.Now().UTC().Add(-50 * time.Second)
 	batchSize := len(records)/4 + 1
 	for i := 0; i < batchSize; i++ {
-		if records[i].Get("updated").(types.DateTime).Time().After(fiftyFiveSecondsAgo) {
+		if records[i].Get("updated").(types.DateTime).Time().After(fiftySecondsAgo) {
 			break
 		}
 		// log.Println("updating", records[i].Get(("name")))

hub/site/src/components/charts/bandwidth-chart.tsx

@@ -72,7 +72,8 @@ export default function BandwidthChart({
 					tickFormatter={hourWithMinutes}
 				/>
 				<ChartTooltip
-					// cursor={false}
+					animationEasing="ease-out"
+					animationDuration={150}
 					content={
 						<ChartTooltipContent
 							unit=" MB/s"

hub/site/src/components/charts/disk-chart.tsx

@@ -77,7 +77,8 @@ export default function DiskChart({
 					tickFormatter={hourWithMinutes}
 				/>
 				<ChartTooltip
-					// cursor={false}
+					animationEasing="ease-out"
+					animationDuration={150}
 					content={
 						<ChartTooltipContent
 							unit=" GB"

hub/site/src/components/charts/disk-io-chart.tsx

@@ -72,7 +72,8 @@ export default function DiskIoChart({
 					tickFormatter={hourWithMinutes}
 				/>
 				<ChartTooltip
-					// cursor={false}
+					animationEasing="ease-out"
+					animationDuration={150}
 					content={
 						<ChartTooltipContent
 							unit=" MB/s"

hub/site/src/components/login/forgot-pass-form.tsx

@@ -86,9 +86,12 @@ export default function ForgotPassword() {
 					<DialogHeader>
 						<DialogTitle>Command line instructions</DialogTitle>
 					</DialogHeader>
-					<p className="text-primary/70 text-[0.95em]">
-						Use the following command to reset
-						your password:
+					<p className="text-primary/70 text-[0.95em] leading-relaxed">
+						If you've lost the password to your admin account, you may reset it using the following
+						command.
+					</p>
+					<p className="text-primary/70 text-[0.95em] leading-relaxed">
+						Then log into the backend and reset your user account password in the users table.
 					</p>
 					<code className="bg-muted rounded-sm py-0.5 px-2.5 mr-auto text-sm">
 						beszel admin update youremail@example.com newpassword

hub/site/src/components/systems-table/systems-table.tsx

@@ -304,7 +304,7 @@ export default function SystemsTable() {
 															? 'auto'
 															: cell.column.getSize(),
 												}}
-												className={'overflow-hidden relative py-3'}
+												className={'overflow-hidden relative py-2.5'}
 											>
 												{flexRender(cell.column.columnDef.cell, cell.getContext())}
 											</TableCell>

readme.md

@@ -2,8 +2,8 @@
 
 A lightweight server resource monitoring hub with historical data, docker stats, and alerts.
 
-[![Docker Image Size (tag)](https://img.shields.io/docker/image-size/henrygd/beszel-agent/0.0.1-alpha.6?logo=docker&label=agent%20image%20size)](https://hub.docker.com/r/henrygd/beszel-agent)
-[![Docker Image Size (tag)](https://img.shields.io/docker/image-size/henrygd/beszel/0.0.1-alpha.6?logo=docker&label=hub%20image%20size)](https://hub.docker.com/r/henrygd/beszel)
+[![Docker Image Size (tag)](https://img.shields.io/docker/image-size/henrygd/beszel-agent/0.0.1-alpha.9?logo=docker&label=agent%20image%20size)](https://hub.docker.com/r/henrygd/beszel-agent)
+[![Docker Image Size (tag)](https://img.shields.io/docker/image-size/henrygd/beszel/0.0.1-alpha.9?logo=docker&label=hub%20image%20size)](https://hub.docker.com/r/henrygd/beszel)
 
 ![Screenshot of the hub](https://henrygd-assets.b-cdn.net/beszel/screenshot.png)
 
@@ -15,7 +15,7 @@ A lightweight server resource monitoring hub with historical data, docker stats,
 - **Multi-user**: Each user has their own systems. Admins can share systems across users.
 - **Simple**: Easy setup and doesn't require anything to be publicly available online.
 - **OAuth / OIDC**: Supports many OAuth2 providers. Password auth can be disabled.
-- **Automated backups**: Automatically back up your data to disk or S3-compatible storage.
+- **Automatic backups**: Save and restore your data to / from disk or S3-compatible storage.
 - **REST API**: Use your metrics in your own scripts and applications.
 
 ## Introduction
@@ -30,7 +30,7 @@ The agent runs on each system you want to monitor. It creates a minimal SSH serv
 
 If using the binary instead of docker, ignore 4-5 and run the agent using the binary instead.
 
-1. Start the hub (see [Installation](#installation)). The binary command is `beszel serve`.
+1. Start the hub (see [installation](#installation)). The binary command is `beszel serve`.
 2. Open http://localhost:8090 and create an admin user.
 3. Click "Add system." Enter the name and host of the system you want to monitor.
 4. Click "Copy docker compose" to copy the agent's docker-compose.yml file to your clipboard.
@@ -95,8 +95,9 @@ Use `beszel update` and `beszel-agent update` to update to the latest version.
 
 Beszel supports OpenID Connect and many OAuth2 authentication providers (see list below). To enable, do the following:
 
-1. Create an OAuth2 application using your provider of choice. The redirect / callback URL should be `<your-beszel-url>/api/oauth2-redirect`.
-2. When you have the client ID and secret, go to the "Auth providers" page and enable your provider.
+1. Make sure your "Application URL" is set correctly in the PocketBase settings.
+2. Create an OAuth2 application using your provider of choice. The redirect / callback URL should be `<your-beszel-url>/api/oauth2-redirect`.
+3. When you have the client ID and secret, go to the "Auth providers" page and enable your provider.
 
 <details>
   <summary>Supported provider list</summary>
@@ -135,7 +136,25 @@ The hub and agent communicate over SSH, so they don't need to be exposed to the
 
 When the hub is started for the first time, it generates an ED25519 key pair.
 
-The agent's SSH server is configured to accept connections only using this key. It does not provide a pty or accept any input, so it is not possible to execute commands on the agent even if your private key is compromised.
+The agent's SSH server is configured to accept connections only using this key. It does not provide a pseudo-terminal or accept input, so it's not possible to execute commands on the agent even if your private key is compromised.
+
+## User roles
+
+### Admin
+
+Assumed to have an admin account in PocketBase, so links to backups, SMTP settings, etc., are shown in the hub.
+
+The first user created automatically becomes an admin and can log into PocketBase.
+
+Please note that changing a user's role will not create a PocketBase admin account for them. If you want to do that, go to Settings > Admins in PocketBase and add them there.
+
+### User
+
+Can create their own systems and alerts. Links to PocketBase settings are not shown in the hub.
+
+### Read only
+
+Cannot create systems, but can view any system that has been shared with them by an admin. Can create alerts.
 
 ## FAQ / Troubleshooting
 
@@ -155,6 +174,10 @@ If it's not set, the agent will try to find the filesystem mounted on `/` and us
 - Run `lsblk` and choose an option under "NAME"
 - Run `sudo fdisk -l` and choose an option under "Device"
 
+### Docker containers are not populating reliably
+
+Try upgrading your docker version on the agent system. I had this issue on a machine running version 24. It was fixed by upgrading to version 27.
+
 ### Month / week records are not populating reliably
 
 Records for longer time periods are made by averaging stats from the shorter time periods. They require the agent to be running uninterrupted for long enough to get a full set of data.