mirror of
https://github.com/henrygd/beszel.git
synced 2026-10-08 06:51:24 +00:00
henrygd
4fe1a0a7a4
Drop the cpu prefix from the cgroup path vars now that memory accounting uses them too, and extract cgroupV2Dir() for the v2 mount lookup previously repeated in CPU, memory and the root cgroup check.
437 lines
15 KiB
Go
437 lines
15 KiB
Go
//go:build testing && linux
|
|
|
|
package agent
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// swapCpuContainerSeams points every container-detection and cgroup path at
|
|
// empty fixtures under a temp dir, then restores them on cleanup.
|
|
func swapCpuContainerSeams(t *testing.T) {
|
|
t.Helper()
|
|
backup := struct {
|
|
root, mountinfo, selfCgroup, systemdCont string
|
|
numCPU func() int
|
|
now func() time.Time
|
|
}{
|
|
cgroupRoot, cgroupMountinfo, procSelfCgroup, cpuSystemdContPath, cpuNumCPU, cpuNow,
|
|
}
|
|
origInLxc := inLxc
|
|
samples := lastCgroupCpuSamples
|
|
env, hadEnv := os.LookupEnv("container")
|
|
t.Cleanup(func() {
|
|
cgroupRoot, cgroupMountinfo, procSelfCgroup, cpuSystemdContPath = backup.root, backup.mountinfo, backup.selfCgroup, backup.systemdCont
|
|
cpuNumCPU, cpuNow = backup.numCPU, backup.now
|
|
inLxc = origInLxc
|
|
lastCgroupCpuSamples = samples
|
|
if hadEnv {
|
|
os.Setenv("container", env)
|
|
}
|
|
})
|
|
|
|
inLxc = sync.OnceValue(detectLxc)
|
|
lastCgroupCpuSamples = make(map[uint16]cgroupCpuSample)
|
|
os.Unsetenv("container")
|
|
t.Setenv("BESZEL_AGENT_USE_CGROUP", "")
|
|
t.Setenv("USE_CGROUP", "")
|
|
|
|
tmp := t.TempDir()
|
|
cgroupRoot = filepath.Join(tmp, "cgroup")
|
|
cgroupMountinfo = filepath.Join(tmp, "mountinfo")
|
|
procSelfCgroup = filepath.Join(tmp, "self-cgroup")
|
|
cpuSystemdContPath = filepath.Join(tmp, "systemd-container")
|
|
}
|
|
|
|
func writeCpuFixture(t *testing.T, path, contents string) {
|
|
t.Helper()
|
|
require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755))
|
|
require.NoError(t, os.WriteFile(path, []byte(contents), 0o644))
|
|
}
|
|
|
|
// markLxc makes detection see a systemd-based LXC guest.
|
|
func markLxc(t *testing.T) {
|
|
t.Helper()
|
|
writeCpuFixture(t, cpuSystemdContPath, "lxc\n")
|
|
}
|
|
|
|
// fakeNow installs a controllable clock and returns a function to advance it.
|
|
func fakeNow(t *testing.T) func(time.Duration) {
|
|
t.Helper()
|
|
cur := time.Unix(1_700_000_000, 0)
|
|
cpuNow = func() time.Time { return cur }
|
|
return func(d time.Duration) { cur = cur.Add(d) }
|
|
}
|
|
|
|
func TestDetectLxc(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
setup func(t *testing.T)
|
|
want bool
|
|
}{
|
|
{"plain host", func(t *testing.T) {}, false},
|
|
{"container env lxc", func(t *testing.T) { t.Setenv("container", "lxc") }, true},
|
|
{"container env podman", func(t *testing.T) { t.Setenv("container", "podman") }, false},
|
|
{"systemd container lxc", func(t *testing.T) { writeCpuFixture(t, cpuSystemdContPath, "lxc\n") }, true},
|
|
{"systemd container nspawn", func(t *testing.T) { writeCpuFixture(t, cpuSystemdContPath, "systemd-nspawn\n") }, false},
|
|
{"lxcfs serving /proc/stat", func(t *testing.T) {
|
|
writeCpuFixture(t, cgroupMountinfo,
|
|
"31 25 0:28 / /proc/stat rw,nosuid,nodev,relatime - fuse.lxcfs lxcfs rw,user_id=0,group_id=0\n")
|
|
}, true},
|
|
// an LXC host (e.g. Proxmox) mounts lxcfs too, but not over its own /proc
|
|
{"lxcfs mounted on host", func(t *testing.T) {
|
|
writeCpuFixture(t, cgroupMountinfo,
|
|
"45 25 0:40 / /var/lib/lxcfs rw,nosuid,nodev,relatime - fuse.lxcfs lxcfs rw,user_id=0,group_id=0\n")
|
|
}, false},
|
|
{"cgroup-only mountinfo", func(t *testing.T) {
|
|
writeCpuFixture(t, cgroupMountinfo,
|
|
"36 25 0:32 / /sys/fs/cgroup rw - cgroup2 cgroup2 rw,nsdelegate\n")
|
|
}, false},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
tt.setup(t)
|
|
assert.Equal(t, tt.want, detectLxc())
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReadCgroupV2CpuSample(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
writeCpuFixture(t, procSelfCgroup, "0::/\n")
|
|
writeCpuFixture(t, cgroupMountinfo, "")
|
|
require.NoError(t, os.MkdirAll(cgroupRoot, 0o755))
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"),
|
|
"usage_usec 3000000\nuser_usec 2000000\nsystem_usec 1000000\nnr_throttled 7\n")
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpuset.cpus.effective"), "2,5-7\n")
|
|
cpuNumCPU = func() int { return 8 }
|
|
|
|
s, ok := readCgroupV2CpuSample()
|
|
require.True(t, ok)
|
|
assert.EqualValues(t, 3000000, s.usageUsec)
|
|
assert.EqualValues(t, 2000000, s.userUsec)
|
|
assert.EqualValues(t, 1000000, s.systemUsec)
|
|
assert.InDelta(t, 4, s.cores, 0.001) // cpuset 2,5-7 = 4 cores
|
|
}
|
|
|
|
// The agent may sit in a sub-cgroup of the guest (a systemd service, or the
|
|
// ".lxc" leaf when started from an attached shell); the mount root still
|
|
// accounts for the whole guest and must win.
|
|
func TestReadCgroupV2PrefersContainerRoot(t *testing.T) {
|
|
for _, rel := range []string{"system.slice/beszel-agent.service", ".lxc"} {
|
|
t.Run(rel, func(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
writeCpuFixture(t, procSelfCgroup, "0::/"+rel+"\n")
|
|
writeCpuFixture(t, cgroupMountinfo, "")
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"), "usage_usec 9000\n")
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, rel, "cpu.stat"), "usage_usec 5\n")
|
|
|
|
s, ok := readCgroupV2CpuSample()
|
|
require.True(t, ok)
|
|
assert.EqualValues(t, 9000, s.usageUsec)
|
|
})
|
|
}
|
|
}
|
|
|
|
// Same for v1: the cpuacct mount root covers the agent's sibling services.
|
|
func TestReadCgroupV1PrefersContainerRoot(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
writeCpuFixture(t, procSelfCgroup, "3:cpu,cpuacct:/system.slice/beszel-agent.service\n")
|
|
v1 := filepath.Join(t.TempDir(), "cpu,cpuacct")
|
|
writeCpuFixture(t, cgroupMountinfo,
|
|
"30 25 0:26 / "+v1+" rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,cpu,cpuacct\n")
|
|
writeCpuFixture(t, filepath.Join(v1, "cpuacct.usage"), "9000000\n")
|
|
writeCpuFixture(t, filepath.Join(v1, "system.slice/beszel-agent.service/cpuacct.usage"), "5000\n")
|
|
|
|
s, ok := readContainerCpuSample()
|
|
require.True(t, ok)
|
|
assert.EqualValues(t, 9000, s.usageUsec)
|
|
}
|
|
|
|
func TestReadCgroupV1CpuSample(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
writeCpuFixture(t, procSelfCgroup, "3:cpuacct:/\n2:memory:/\n")
|
|
v1 := filepath.Join(t.TempDir(), "cpuacct")
|
|
writeCpuFixture(t, cgroupMountinfo,
|
|
"30 25 0:26 / "+v1+" rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,cpuacct\n")
|
|
writeCpuFixture(t, filepath.Join(v1, "cpuacct.usage"), "2000000000\n")
|
|
writeCpuFixture(t, filepath.Join(v1, "cpuacct.stat"), "user 100\nsystem 50\n")
|
|
cpuNumCPU = func() int { return 4 }
|
|
|
|
s, ok := readContainerCpuSample() // no 0:: line -> falls through to v1
|
|
require.True(t, ok)
|
|
assert.EqualValues(t, 2000000, s.usageUsec) // ns -> usec
|
|
assert.EqualValues(t, 1000000, s.userUsec) // 100 jiffies * 1e6/100
|
|
assert.EqualValues(t, 500000, s.systemUsec) // 50 jiffies
|
|
assert.InDelta(t, 4, s.cores, 0.001)
|
|
}
|
|
|
|
func TestContainerCpuMetricsMath(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
markLxc(t)
|
|
writeCpuFixture(t, procSelfCgroup, "0::/\n")
|
|
writeCpuFixture(t, cgroupMountinfo, "")
|
|
require.NoError(t, os.MkdirAll(cgroupRoot, 0o755))
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpuset.cpus.effective"), "0-3\n")
|
|
cpuNumCPU = func() int { return 8 }
|
|
advance := fakeNow(t)
|
|
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"),
|
|
"usage_usec 1000000\nuser_usec 600000\nsystem_usec 400000\n")
|
|
m, ok := containerCpuMetrics(60000, false)
|
|
require.True(t, ok)
|
|
assert.Zero(t, m.Total) // first call only seeds the baseline
|
|
|
|
// 1s elapsed, container burned 2 core-seconds on 4 usable cores
|
|
advance(time.Second)
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"),
|
|
"usage_usec 3000000\nuser_usec 1600000\nsystem_usec 900000\n")
|
|
m, ok = containerCpuMetrics(60000, false)
|
|
require.True(t, ok)
|
|
assert.InDelta(t, 50, m.Total, 0.01)
|
|
assert.InDelta(t, 25, m.User, 0.01)
|
|
assert.InDelta(t, 12.5, m.System, 0.01)
|
|
assert.Zero(t, m.Iowait)
|
|
assert.Zero(t, m.Steal)
|
|
assert.InDelta(t, 50, m.Idle, 0.01)
|
|
}
|
|
|
|
func TestContainerCpuMetricsHonorsQuota(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
markLxc(t)
|
|
writeCpuFixture(t, procSelfCgroup, "0::/\n")
|
|
writeCpuFixture(t, cgroupMountinfo, "")
|
|
require.NoError(t, os.MkdirAll(cgroupRoot, 0o755))
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.max"), "200000 100000\n") // 2 cores
|
|
cpuNumCPU = func() int { return 8 }
|
|
advance := fakeNow(t)
|
|
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"), "usage_usec 1000000\n")
|
|
containerCpuMetrics(60000, false)
|
|
advance(time.Second)
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"), "usage_usec 2000000\n")
|
|
m, ok := containerCpuMetrics(60000, false)
|
|
require.True(t, ok)
|
|
assert.InDelta(t, 50, m.Total, 0.01) // 1 core-second against a 2-core quota
|
|
}
|
|
|
|
func TestContainerCpuMetricsZeroAndBackwardDelta(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
markLxc(t)
|
|
writeCpuFixture(t, procSelfCgroup, "0::/\n")
|
|
writeCpuFixture(t, cgroupMountinfo, "")
|
|
require.NoError(t, os.MkdirAll(cgroupRoot, 0o755))
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"), "usage_usec 5000000\n")
|
|
cpuNumCPU = func() int { return 4 }
|
|
advance := fakeNow(t)
|
|
|
|
// seed the baseline, then do not advance the clock: elapsed <= 0
|
|
containerCpuMetrics(60000, false)
|
|
m, ok := containerCpuMetrics(60000, false)
|
|
require.True(t, ok)
|
|
assert.Zero(t, m.Total)
|
|
|
|
// counter goes backwards (cgroup recreated): report zero and re-baseline
|
|
advance(time.Second)
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"), "usage_usec 100000\n")
|
|
m, ok = containerCpuMetrics(60000, false)
|
|
require.True(t, ok)
|
|
assert.Zero(t, m.Total)
|
|
|
|
// next tick measures from the new baseline, not the stale one
|
|
advance(time.Second)
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"), "usage_usec 1100000\n")
|
|
m, ok = containerCpuMetrics(60000, false)
|
|
require.True(t, ok)
|
|
assert.InDelta(t, 25, m.Total, 0.01) // 1e6 usec / (1s * 4 cores)
|
|
}
|
|
|
|
func TestContainerCpuMetricsFallbacks(t *testing.T) {
|
|
t.Run("not in lxc", func(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
_, ok := containerCpuMetrics(60000, false)
|
|
assert.False(t, ok)
|
|
})
|
|
// Docker agents monitor the host, so cgroup accounting must not kick in
|
|
// even when it is readable and no LXC marker is present.
|
|
t.Run("docker container", func(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
writeCpuFixture(t, procSelfCgroup, "0::/\n")
|
|
writeCpuFixture(t, cgroupMountinfo, "")
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"), "usage_usec 5000000\n")
|
|
_, ok := containerCpuMetrics(60000, false)
|
|
assert.False(t, ok)
|
|
})
|
|
t.Run("in lxc without cgroup accounting", func(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
markLxc(t)
|
|
writeCpuFixture(t, procSelfCgroup, "0::/\n")
|
|
writeCpuFixture(t, cgroupMountinfo, "")
|
|
// cgroupRoot has no cpu.stat
|
|
_, ok := containerCpuMetrics(60000, false)
|
|
assert.False(t, ok)
|
|
})
|
|
}
|
|
|
|
func TestContainerCpuMetricsExplicitOptIn(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
writeCpuFixture(t, procSelfCgroup, "0::/system.slice/beszel-agent.service\n")
|
|
writeCpuFixture(t, cgroupMountinfo, "")
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"), "usage_usec 1000000\n")
|
|
cpuNumCPU = func() int { return 4 }
|
|
advance := fakeNow(t)
|
|
|
|
a := &Agent{forceUseCgroup: true}
|
|
a.initializeCpu()
|
|
require.False(t, lastCgroupCpuSamples[60000].at.IsZero())
|
|
advance(time.Second)
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"), "usage_usec 3000000\n")
|
|
m, err := getCpuMetrics(60000, a.forceUseCgroup)
|
|
require.NoError(t, err)
|
|
assert.InDelta(t, 50, m.Total, 0.01)
|
|
assert.True(t, m.fromCgroup)
|
|
}
|
|
|
|
func TestContainerCpuMetricsExplicitOptInUnreadable(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
writeCpuFixture(t, procSelfCgroup, "0::/\n")
|
|
_, ok := containerCpuMetrics(60000, true)
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestInRootCgroupV2(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
writeCpuFixture(t, cgroupMountinfo, "")
|
|
|
|
// no cgroup v2 membership
|
|
writeCpuFixture(t, procSelfCgroup, "4:cpu,cpuacct:/docker/abc\n")
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cgroup.controllers"), "cpu memory\n")
|
|
assert.False(t, inRootCgroupV2())
|
|
|
|
// --cgroupns=host: the mount is the host root, which has no cgroup.type
|
|
writeCpuFixture(t, procSelfCgroup, "0::/system.slice/docker-abc.scope\n")
|
|
assert.True(t, inRootCgroupV2())
|
|
|
|
// --cgroupns=private: the mount is the container's own (non-root) cgroup
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cgroup.type"), "domain\n")
|
|
assert.False(t, inRootCgroupV2())
|
|
|
|
// no cgroup v2 mount at all
|
|
require.NoError(t, os.RemoveAll(cgroupRoot))
|
|
assert.False(t, inRootCgroupV2())
|
|
}
|
|
|
|
func TestInitializeCpuDetectsLxc(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
markLxc(t)
|
|
writeCpuFixture(t, procSelfCgroup, "0::/\n")
|
|
writeCpuFixture(t, cgroupMountinfo, "")
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"), "usage_usec 1000000\n")
|
|
|
|
a := &Agent{}
|
|
a.initializeCpu()
|
|
assert.EqualValues(t, 1000000, lastCgroupCpuSamples[60000].usageUsec)
|
|
}
|
|
|
|
func TestNewAgentInitializesCgroupCpuFromEnv(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
t.Setenv("BESZEL_AGENT_USE_CGROUP", "true")
|
|
writeCpuFixture(t, procSelfCgroup, "0::/system.slice/beszel-agent.service\n")
|
|
writeCpuFixture(t, cgroupMountinfo, "")
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"), "usage_usec 1000000\n")
|
|
cpuNumCPU = func() int { return 4 }
|
|
advance := fakeNow(t)
|
|
|
|
a, err := NewAgent(t.TempDir())
|
|
require.NoError(t, err)
|
|
require.True(t, a.forceUseCgroup)
|
|
assert.EqualValues(t, 1000000, lastCgroupCpuSamples[60000].usageUsec)
|
|
|
|
// Changing the environment later must not change this agent's CPU source.
|
|
t.Setenv("BESZEL_AGENT_USE_CGROUP", "false")
|
|
advance(time.Second)
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"), "usage_usec 3000000\n")
|
|
m, err := getCpuMetrics(60000, a.forceUseCgroup)
|
|
require.NoError(t, err)
|
|
assert.InDelta(t, 50, m.Total, 0.01)
|
|
assert.True(t, m.fromCgroup)
|
|
}
|
|
|
|
// The host path must keep reporting through gopsutil untouched.
|
|
func TestGetCpuMetricsHostFallback(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
m, err := getCpuMetrics(60000, false)
|
|
require.NoError(t, err)
|
|
assert.False(t, m.fromCgroup)
|
|
assert.GreaterOrEqual(t, m.Total, 0.0)
|
|
assert.LessOrEqual(t, m.Total, 100.0)
|
|
}
|
|
|
|
// Inside LXC getCpuMetrics must report the cgroup-derived value, not
|
|
// the host core counters from /proc/stat.
|
|
func TestGetCpuMetricsPrefersCgroup(t *testing.T) {
|
|
swapCpuContainerSeams(t)
|
|
markLxc(t)
|
|
writeCpuFixture(t, procSelfCgroup, "0::/\n")
|
|
writeCpuFixture(t, cgroupMountinfo, "")
|
|
require.NoError(t, os.MkdirAll(cgroupRoot, 0o755))
|
|
cpuNumCPU = func() int { return 4 }
|
|
advance := fakeNow(t)
|
|
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"), "usage_usec 0\n")
|
|
m, err := getCpuMetrics(60000, false)
|
|
require.NoError(t, err)
|
|
assert.Zero(t, m.Total)
|
|
|
|
advance(time.Second)
|
|
writeCpuFixture(t, filepath.Join(cgroupRoot, "cpu.stat"), "usage_usec 2000000\n")
|
|
m, err = getCpuMetrics(60000, false)
|
|
require.NoError(t, err)
|
|
assert.InDelta(t, 50, m.Total, 0.01)
|
|
assert.True(t, m.fromCgroup) // per-core usage is skipped for this source
|
|
}
|
|
|
|
func TestCountCpuList(t *testing.T) {
|
|
assert.Equal(t, 4, countCpuList("0-3"))
|
|
assert.Equal(t, 4, countCpuList("2,5-7"))
|
|
assert.Equal(t, 1, countCpuList("2"))
|
|
assert.Equal(t, 0, countCpuList(""))
|
|
assert.Equal(t, 0, countCpuList("max"))
|
|
assert.Equal(t, 6, countCpuList("0-3,8-9"))
|
|
}
|
|
|
|
func TestCpuQuotaCores(t *testing.T) {
|
|
dir := t.TempDir()
|
|
_, ok := cpuQuotaCores(dir)
|
|
assert.False(t, ok) // no quota files
|
|
|
|
writeCpuFixture(t, filepath.Join(dir, "cpu.max"), "max 100000\n")
|
|
_, ok = cpuQuotaCores(dir)
|
|
assert.False(t, ok) // unlimited
|
|
|
|
writeCpuFixture(t, filepath.Join(dir, "cpu.max"), "150000 100000\n")
|
|
q, ok := cpuQuotaCores(dir)
|
|
require.True(t, ok)
|
|
assert.InDelta(t, 1.5, q, 0.001)
|
|
|
|
// v1 files
|
|
v1 := t.TempDir()
|
|
writeCpuFixture(t, filepath.Join(v1, "cpu.cfs_quota_us"), "-1\n")
|
|
writeCpuFixture(t, filepath.Join(v1, "cpu.cfs_period_us"), "100000\n")
|
|
_, ok = cpuQuotaCores(v1)
|
|
assert.False(t, ok)
|
|
|
|
writeCpuFixture(t, filepath.Join(v1, "cpu.cfs_quota_us"), "50000\n")
|
|
q, ok = cpuQuotaCores(v1)
|
|
require.True(t, ok)
|
|
assert.InDelta(t, 0.5, q, 0.001)
|
|
}
|