0.0.1-alpha.9

subtract mem cache from container stats

agent/main.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"sync"
 	"time"
 
 	sshServer "github.com/gliderlabs/ssh"
@@ -21,11 +22,20 @@ import (
 	psutilNet "github.com/shirou/gopsutil/v4/net"
 )
 
-var Version = "0.0.1-alpha.7"
+var Version = "0.0.1-alpha.9"
 
 var containerCpuMap = make(map[string][2]uint64)
+var containerCpuMutex = &sync.Mutex{}
 
-// var containerCpuMutex = &sync.Mutex{}
+var sem = make(chan struct{}, 15)
+
+func acquireSemaphore() {
+	sem <- struct{}{}
+}
+
+func releaseSemaphore() {
+	<-sem
+}
 
 var diskIoStats = DiskIoStats{
 	Read:       0,
@@ -48,54 +58,15 @@ var client = &http.Client{
 		Dial: func(proto, addr string) (net.Conn, error) {
 			return net.Dial("unix", "/var/run/docker.sock")
 		},
-		ForceAttemptHTTP2:  false,
-		IdleConnTimeout:    90 * time.Second,
-		DisableCompression: true,
-		MaxIdleConns:       10,
-		DisableKeepAlives:  false,
+		ForceAttemptHTTP2:   false,
+		IdleConnTimeout:     90 * time.Second,
+		DisableCompression:  true,
+		MaxIdleConnsPerHost: 50,
+		DisableKeepAlives:   false,
 	},
 }
 
-type SystemData struct {
-	Stats      SystemStats      `json:"stats"`
-	Info       SystemInfo       `json:"info"`
-	Containers []ContainerStats `json:"container"`
-}
-
-type SystemInfo struct {
-	Cores    int    `json:"c"`
-	Threads  int    `json:"t"`
-	CpuModel string `json:"m"`
-	// Os       string  `json:"o"`
-	Uptime  uint64  `json:"u"`
-	Cpu     float64 `json:"cpu"`
-	MemPct  float64 `json:"mp"`
-	DiskPct float64 `json:"dp"`
-}
-
-type SystemStats struct {
-	Cpu          float64 `json:"cpu"`
-	Mem          float64 `json:"m"`
-	MemUsed      float64 `json:"mu"`
-	MemPct       float64 `json:"mp"`
-	MemBuffCache float64 `json:"mb"`
-	Disk         float64 `json:"d"`
-	DiskUsed     float64 `json:"du"`
-	DiskPct      float64 `json:"dp"`
-	DiskRead     float64 `json:"dr"`
-	DiskWrite    float64 `json:"dw"`
-	NetworkSent  float64 `json:"ns"`
-	NetworkRecv  float64 `json:"nr"`
-}
-
-type ContainerStats struct {
-	Name string  `json:"n"`
-	Cpu  float64 `json:"c"`
-	Mem  float64 `json:"m"`
-	// MemPct float64 `json:"mp"`
-}
-
-func getSystemStats() (SystemInfo, SystemStats) {
+func getSystemStats() (*SystemInfo, *SystemStats) {
 	c, _ := cpu.Percent(0, false)
 	v, _ := mem.VirtualMemory()
 	d, _ := disk.Usage("/")
@@ -104,7 +75,7 @@ func getSystemStats() (SystemInfo, SystemStats) {
 	memPct := twoDecimals(v.UsedPercent)
 	diskPct := twoDecimals(d.UsedPercent)
 
-	systemStats := SystemStats{
+	systemStats := &SystemStats{
 		Cpu:          cpuPct,
 		Mem:          bytesToGigabytes(v.Total),
 		MemUsed:      bytesToGigabytes(v.Used),
@@ -115,7 +86,7 @@ func getSystemStats() (SystemInfo, SystemStats) {
 		DiskPct:      diskPct,
 	}
 
-	systemInfo := SystemInfo{
+	systemInfo := &SystemInfo{
 		Cpu:     cpuPct,
 		MemPct:  memPct,
 		DiskPct: diskPct,
@@ -181,51 +152,63 @@ func getSystemStats() (SystemInfo, SystemStats) {
 
 }
 
-func getDockerStats() ([]ContainerStats, error) {
+func getDockerStats() ([]*ContainerStats, error) {
 	resp, err := client.Get("http://localhost/containers/json")
 	if err != nil {
-		return []ContainerStats{}, err
+		return []*ContainerStats{}, err
 	}
 	defer resp.Body.Close()
 
-	var containers []Container
+	var containers []*Container
 	if err := json.NewDecoder(resp.Body).Decode(&containers); err != nil {
 		panic(err)
 	}
 
-	var containerStats []ContainerStats
+	containerStats := make([]*ContainerStats, 0, len(containers))
+
+	// store valid ids to clean up old container ids from map
+	validIds := make(map[string]struct{}, len(containers))
+
+	var wg sync.WaitGroup
 
 	for _, ctr := range containers {
-		cstats, err := getContainerStats(ctr)
-		if err != nil {
-			// retry once
-			cstats, err = getContainerStats(ctr)
+		ctr.IdShort = ctr.ID[:12]
+		validIds[ctr.IdShort] = struct{}{}
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			cstats, err := getContainerStats(ctr)
 			if err != nil {
-				log.Printf("Error getting container stats: %+v\n", err)
-				continue
+				// retry once
+				cstats, err = getContainerStats(ctr)
+				if err != nil {
+					log.Printf("Error getting container stats: %+v\n", err)
+					return
+				}
 			}
-		}
-		containerStats = append(containerStats, cstats)
+			containerStats = append(containerStats, cstats)
+		}()
 	}
 
-	// clean up old containers from map
-	validNames := make(map[string]struct{}, len(containers))
-	for _, ctr := range containers {
-		validNames[ctr.Names[0][1:]] = struct{}{}
-	}
-	for name := range containerCpuMap {
-		if _, exists := validNames[name]; !exists {
-			delete(containerCpuMap, name)
+	wg.Wait()
+
+	for id := range containerCpuMap {
+		if _, exists := validIds[id]; !exists {
+			// log.Printf("Removing container cpu map entry: %+v\n", id)
+			delete(containerCpuMap, id)
 		}
 	}
 
 	return containerStats, nil
 }
 
-func getContainerStats(ctr Container) (ContainerStats, error) {
-	resp, err := client.Get("http://localhost/containers/" + ctr.ID + "/stats?stream=0&one-shot=1")
+func getContainerStats(ctr *Container) (*ContainerStats, error) {
+	// use semaphore to limit concurrency
+	acquireSemaphore()
+	defer releaseSemaphore()
+	resp, err := client.Get("http://localhost/containers/" + ctr.IdShort + "/stats?stream=0&one-shot=1")
 	if err != nil {
-		return ContainerStats{}, err
+		return &ContainerStats{}, err
 	}
 	defer resp.Body.Close()
 
@@ -236,26 +219,30 @@ func getContainerStats(ctr Container) (ContainerStats, error) {
 
 	name := ctr.Names[0][1:]
 
-	// memory
-	usedMemory := statsJson.MemoryStats.Usage - statsJson.MemoryStats.Cache
+	// memory (https://docs.docker.com/reference/cli/docker/container/stats/)
+	memCache := statsJson.MemoryStats.Stats["inactive_file"]
+	if memCache == 0 {
+		memCache = statsJson.MemoryStats.Stats["cache"]
+	}
+	usedMemory := statsJson.MemoryStats.Usage - memCache
 	// pctMemory := float64(usedMemory) / float64(statsJson.MemoryStats.Limit) * 100
 
 	// cpu
 	// add default values to containerCpu if it doesn't exist
-	// containerCpuMutex.Lock()
-	// defer containerCpuMutex.Unlock()
-	if _, ok := containerCpuMap[name]; !ok {
-		containerCpuMap[name] = [2]uint64{0, 0}
+	containerCpuMutex.Lock()
+	defer containerCpuMutex.Unlock()
+	if _, ok := containerCpuMap[ctr.IdShort]; !ok {
+		containerCpuMap[ctr.IdShort] = [2]uint64{0, 0}
 	}
-	cpuDelta := statsJson.CPUStats.CPUUsage.TotalUsage - containerCpuMap[name][0]
-	systemDelta := statsJson.CPUStats.SystemUsage - containerCpuMap[name][1]
+	cpuDelta := statsJson.CPUStats.CPUUsage.TotalUsage - containerCpuMap[ctr.IdShort][0]
+	systemDelta := statsJson.CPUStats.SystemUsage - containerCpuMap[ctr.IdShort][1]
 	cpuPct := float64(cpuDelta) / float64(systemDelta) * 100
 	if cpuPct > 100 {
-		return ContainerStats{}, fmt.Errorf("%s cpu pct greater than 100: %+v", name, cpuPct)
+		return &ContainerStats{}, fmt.Errorf("%s cpu pct greater than 100: %+v", name, cpuPct)
 	}
-	containerCpuMap[name] = [2]uint64{statsJson.CPUStats.CPUUsage.TotalUsage, statsJson.CPUStats.SystemUsage}
+	containerCpuMap[ctr.IdShort] = [2]uint64{statsJson.CPUStats.CPUUsage.TotalUsage, statsJson.CPUStats.SystemUsage}
 
-	cStats := ContainerStats{
+	cStats := &ContainerStats{
 		Name: name,
 		Cpu:  twoDecimals(cpuPct),
 		Mem:  bytesToMegabytes(float64(usedMemory)),
@@ -264,12 +251,12 @@ func getContainerStats(ctr Container) (ContainerStats, error) {
 	return cStats, nil
 }
 
-func gatherStats() SystemData {
+func gatherStats() *SystemData {
 	systemInfo, systemStats := getSystemStats()
-	stats := SystemData{
+	stats := &SystemData{
 		Stats:      systemStats,
 		Info:       systemInfo,
-		Containers: []ContainerStats{},
+		Containers: []*ContainerStats{},
 	}
 	containerStats, err := getDockerStats()
 	if err == nil {

agent/types.go

@@ -2,8 +2,48 @@ package main
 
 import "time"
 
+type SystemData struct {
+	Stats      *SystemStats      `json:"stats"`
+	Info       *SystemInfo       `json:"info"`
+	Containers []*ContainerStats `json:"container"`
+}
+
+type SystemInfo struct {
+	Cores    int    `json:"c"`
+	Threads  int    `json:"t"`
+	CpuModel string `json:"m"`
+	// Os       string  `json:"o"`
+	Uptime  uint64  `json:"u"`
+	Cpu     float64 `json:"cpu"`
+	MemPct  float64 `json:"mp"`
+	DiskPct float64 `json:"dp"`
+}
+
+type SystemStats struct {
+	Cpu          float64 `json:"cpu"`
+	Mem          float64 `json:"m"`
+	MemUsed      float64 `json:"mu"`
+	MemPct       float64 `json:"mp"`
+	MemBuffCache float64 `json:"mb"`
+	Disk         float64 `json:"d"`
+	DiskUsed     float64 `json:"du"`
+	DiskPct      float64 `json:"dp"`
+	DiskRead     float64 `json:"dr"`
+	DiskWrite    float64 `json:"dw"`
+	NetworkSent  float64 `json:"ns"`
+	NetworkRecv  float64 `json:"nr"`
+}
+
+type ContainerStats struct {
+	Name string  `json:"n"`
+	Cpu  float64 `json:"c"`
+	Mem  float64 `json:"m"`
+	// MemPct float64 `json:"mp"`
+}
+
 type Container struct {
 	ID      string `json:"Id"`
+	IdShort string
 	Names   []string
 	Image   string
 	ImageID string

hub/main.go

@@ -30,7 +30,7 @@ import (
 	"golang.org/x/crypto/ssh"
 )
 
-var Version = "0.0.1-alpha.7"
+var Version = "0.0.1-alpha.9"
 
 var app *pocketbase.PocketBase
 var serverConnections = make(map[string]*Server)
@@ -217,14 +217,14 @@ func updateSystems() {
 		0,                    // offset
 	)
 	// log.Println("records", len(records))
-	if err != nil {
-		app.Logger().Error("Failed to query systems: ", "err", err.Error())
+	if err != nil || len(records) == 0 {
+		// app.Logger().Error("Failed to query systems")
 		return
 	}
-	fiftyFiveSecondsAgo := time.Now().UTC().Add(-55 * time.Second)
+	fiftySecondsAgo := time.Now().UTC().Add(-50 * time.Second)
 	batchSize := len(records)/4 + 1
 	for i := 0; i < batchSize; i++ {
-		if records[i].Get("updated").(types.DateTime).Time().After(fiftyFiveSecondsAgo) {
+		if records[i].Get("updated").(types.DateTime).Time().After(fiftySecondsAgo) {
 			break
 		}
 		// log.Println("updating", records[i].Get(("name")))

hub/site/src/components/charts/bandwidth-chart.tsx

@@ -72,7 +72,8 @@ export default function BandwidthChart({
 					tickFormatter={hourWithMinutes}
 				/>
 				<ChartTooltip
-					// cursor={false}
+					animationEasing="ease-out"
+					animationDuration={150}
 					content={
 						<ChartTooltipContent
 							unit=" MB/s"

hub/site/src/components/charts/disk-chart.tsx

@@ -77,7 +77,8 @@ export default function DiskChart({
 					tickFormatter={hourWithMinutes}
 				/>
 				<ChartTooltip
-					// cursor={false}
+					animationEasing="ease-out"
+					animationDuration={150}
 					content={
 						<ChartTooltipContent
 							unit=" GB"

hub/site/src/components/charts/disk-io-chart.tsx

@@ -72,7 +72,8 @@ export default function DiskIoChart({
 					tickFormatter={hourWithMinutes}
 				/>
 				<ChartTooltip
-					// cursor={false}
+					animationEasing="ease-out"
+					animationDuration={150}
 					content={
 						<ChartTooltipContent
 							unit=" MB/s"

hub/site/src/components/systems-table/systems-table.tsx

@@ -304,7 +304,7 @@ export default function SystemsTable() {
 															? 'auto'
 															: cell.column.getSize(),
 												}}
-												className={'overflow-hidden relative py-3'}
+												className={'overflow-hidden relative py-2.5'}
 											>
 												{flexRender(cell.column.columnDef.cell, cell.getContext())}
 											</TableCell>

readme.md

@@ -95,8 +95,9 @@ Use `beszel update` and `beszel-agent update` to update to the latest version.
 
 Beszel supports OpenID Connect and many OAuth2 authentication providers (see list below). To enable, do the following:
 
-1. Create an OAuth2 application using your provider of choice. The redirect / callback URL should be `<your-beszel-url>/api/oauth2-redirect`.
-2. When you have the client ID and secret, go to the "Auth providers" page and enable your provider.
+1. Make sure your "Application URL" is set correctly in the PocketBase settings.
+2. Create an OAuth2 application using your provider of choice. The redirect / callback URL should be `<your-beszel-url>/api/oauth2-redirect`.
+3. When you have the client ID and secret, go to the "Auth providers" page and enable your provider.
 
 <details>
   <summary>Supported provider list</summary>
@@ -137,6 +138,24 @@ When the hub is started for the first time, it generates an ED25519 key pair.
 
 The agent's SSH server is configured to accept connections only using this key. It does not provide a pty or accept any input, so it is not possible to execute commands on the agent even if your private key is compromised.
 
+## User roles
+
+### Admin
+
+Assumed to have an admin account in PocketBase, so links to backups, SMTP settings, etc., are shown in the hub.
+
+The first user created automatically becomes an admin and can log into PocketBase.
+
+Please note that changing a user's role will not create a PocketBase admin account for them. If you want to do that, go to Settings > Admins in PocketBase and add them there.
+
+### User
+
+Can create their own systems and alerts. Links to PocketBase settings are not shown in the hub.
+
+### Read only
+
+Cannot create systems, but can view any system that has been shared with them by an admin. Can create alerts.
+
 ## FAQ / Troubleshooting
 
 ### Agent is not connecting
@@ -155,6 +174,10 @@ If it's not set, the agent will try to find the filesystem mounted on `/` and us
 - Run `lsblk` and choose an option under "NAME"
 - Run `sudo fdisk -l` and choose an option under "Device"
 
+### Docker containers are not populating reliably
+
+Try upgrading your docker version on the agent system. I had this issue on a machine running docker 24. It was fixed by upgrading to version 27.
+
 ### Month / week records are not populating reliably
 
 Records for longer time periods are made by averaging stats from the shorter time periods. They require the agent to be running uninterrupted for long enough to get a full set of data.