From 53021bec3606d4ca28e45c5f1d57461b652bb23a Mon Sep 17 00:00:00 2001 From: Danny Canter Date: Tue, 19 Aug 2025 17:31:51 -0700 Subject: [PATCH] Cgroup2Manager: Various adjustments (#270) Systemd loves to move the root cgroups processes to one it created named /init.scope and then write the root cgs subtree_control file. Because of this we can't just add exec processes to the cg we made for the container anymore as we'll get EBUSY. We should follow where the init processes cg is actually at (/proc/pid/cgroup) and add it there. --- vminitd/Sources/vmexec/ExecCommand.swift | 22 +++-- ...roupManager.swift => Cgroup2Manager.swift} | 89 ++++++++++++++++--- .../Sources/vminitd/ManagedContainer.swift | 60 +++++++------ vminitd/Sources/vminitd/ManagedProcess.swift | 19 ++-- vminitd/Sources/vminitd/Server+GRPC.swift | 8 +- 5 files changed, 142 insertions(+), 56 deletions(-) rename vminitd/Sources/vminitd/{CgroupManager.swift => Cgroup2Manager.swift} (63%) diff --git a/vminitd/Sources/vmexec/ExecCommand.swift b/vminitd/Sources/vmexec/ExecCommand.swift index e26e4b86..501e118c 100644 --- a/vminitd/Sources/vmexec/ExecCommand.swift +++ b/vminitd/Sources/vmexec/ExecCommand.swift @@ -46,14 +46,8 @@ struct ExecCommand: ParsableCommand { try execInNamespaces(process: process, log: log) } - static func enterNS(path: String, nsType: Int32) throws { - let fd = open(path, O_RDONLY) - if fd <= 0 { - throw App.Errno(stage: "open(ns)") - } - defer { close(fd) } - - guard setns(fd, nsType) == 0 else { + static func enterNS(pidFd: Int32, nsType: Int32) throws { + guard setns(pidFd, nsType) == 0 else { throw App.Errno(stage: "setns(fd)") } } @@ -65,10 +59,14 @@ struct ExecCommand: ParsableCommand { let syncPipe = FileHandle(fileDescriptor: 3) let ackPipe = FileHandle(fileDescriptor: 4) - try Self.enterNS(path: "/proc/\(self.parentPid)/ns/cgroup", nsType: CLONE_NEWCGROUP) - try Self.enterNS(path: "/proc/\(self.parentPid)/ns/pid", nsType: CLONE_NEWPID) - try Self.enterNS(path: "/proc/\(self.parentPid)/ns/uts", nsType: CLONE_NEWUTS) - try Self.enterNS(path: "/proc/\(self.parentPid)/ns/mnt", nsType: CLONE_NEWNS) + let pidFd = CZ_pidfd_open(Int32(parentPid), 0) + guard pidFd > 0 else { + throw App.Errno(stage: "pidfd_open(\(parentPid))") + } + try Self.enterNS( + pidFd: pidFd, + nsType: CLONE_NEWCGROUP | CLONE_NEWPID | CLONE_NEWUTS | CLONE_NEWNS + ) let processID = fork() diff --git a/vminitd/Sources/vminitd/CgroupManager.swift b/vminitd/Sources/vminitd/Cgroup2Manager.swift similarity index 63% rename from vminitd/Sources/vminitd/CgroupManager.swift rename to vminitd/Sources/vminitd/Cgroup2Manager.swift index de745d11..ae02a1c3 100644 --- a/vminitd/Sources/vminitd/CgroupManager.swift +++ b/vminitd/Sources/vminitd/Cgroup2Manager.swift @@ -14,10 +14,17 @@ // limitations under the License. //===----------------------------------------------------------------------===// +#if os(Linux) + +#if canImport(Musl) +import Musl +#elseif canImport(Glibc) +import Glibc +#endif + import ContainerizationOS import Foundation import Logging -import Musl enum Cgroup2Controller: String { case pids @@ -30,27 +37,71 @@ enum Cgroup2Controller: String { // Extremely simple cgroup manager. Our needs are simple for now, and this is // reflected in the type. -internal struct Cgroup2Manager { +struct Cgroup2Manager: Sendable { static let defaultMountPoint = URL(filePath: "/sys/fs/cgroup") - static let killFile = "cgroup.kill" - static let procsFile = "cgroup.procs" - static let subtreeControlFile = "cgroup.subtree_control" + private static let killFile = "cgroup.kill" + private static let procsFile = "cgroup.procs" + private static let subtreeControlFile = "cgroup.subtree_control" + + private static let cg2Magic = 0x6367_7270 private let mountPoint: URL private let path: URL private let logger: Logger? init( - mountPoint: URL = defaultMountPoint, - path: URL, - perms: Int16 = 0o755, + mountPoint: URL = Self.defaultMountPoint, + group: URL, logger: Logger? = nil - ) throws { + ) { self.mountPoint = mountPoint - self.path = mountPoint.appending(path: path.path) + self.path = mountPoint.appending(path: group.path) self.logger = logger + } + static func load( + mountPoint: URL = Self.defaultMountPoint, + group: URL, + logger: Logger? = nil + ) throws -> Cgroup2Manager { + let path = mountPoint.appending(path: group.path) + var s = statfs() + let res = statfs(path.path, &s) + if res != 0 { + throw Error.errno(errno: errno, message: "failed to statfs \(path.path)") + } + if Int64(s.f_type) != Self.cg2Magic { + throw Error.notCgroup + } + return Cgroup2Manager( + mountPoint: mountPoint, + group: group, + logger: logger + ) + } + + static func loadFromPid(pid: Int32, logger: Logger? = nil) throws -> Cgroup2Manager { + let procCgPath = URL(filePath: "/proc/\(pid)/cgroup") + let fh = try FileHandle(forReadingFrom: procCgPath) + guard let data = try fh.readToEnd() else { + throw Error.errno(errno: errno, message: "failed to read \(procCgPath)") + } + + // If this fails we have bigger problems. + let str = String(data: data, encoding: .utf8)! + let parts = str.split(separator: ":") + if parts[0] != "0" { + throw Error.cgroup1 + } + + // We should really read /proc/pid/mountinfo, but for now just assume + // it's always at /sys/fs/cgroup. + let path = parts[1].trimmingCharacters(in: .whitespacesAndNewlines) + return Cgroup2Manager(group: URL(filePath: String(path)), logger: logger) + } + + func create(perms: Int16 = 0o755) throws { self.logger?.info( "creating cgroup manager", metadata: [ @@ -110,6 +161,13 @@ internal struct Cgroup2Manager { } func addProcess(pid: Int32) throws { + self.logger?.debug( + "adding new proc to cgroup", + metadata: [ + "mountpoint": "\(self.mountPoint.path)", + "path": "\(self.path.path)", + ]) + let pidStr = String(pid) try Self.writeValue( path: self.path, @@ -143,13 +201,24 @@ internal struct Cgroup2Manager { extension Cgroup2Manager { enum Error: Swift.Error, CustomStringConvertible { + case notCgroup + case cgroup1 case errno(errno: Int32, message: String) + case notExist(path: String) var description: String { switch self { case .errno(let errno, let message): return "failed with errno \(errno): \(message)" + case .notExist(let path): + return "cgroup at path \(path) does not exist" + case .cgroup1: + return "tried to load a cgroup v1 path" + case .notCgroup: + return "path is not a cgroup mountpoint" } } } } + +#endif diff --git a/vminitd/Sources/vminitd/ManagedContainer.swift b/vminitd/Sources/vminitd/ManagedContainer.swift index ed15a2ae..29b9ce4e 100644 --- a/vminitd/Sources/vminitd/ManagedContainer.swift +++ b/vminitd/Sources/vminitd/ManagedContainer.swift @@ -39,12 +39,6 @@ actor ManagedContainer { spec: ContainerizationOCI.Spec, log: Logger ) throws { - let bundle = try ContainerizationOCI.Bundle.create( - path: Self.craftBundlePath(id: id), - spec: spec - ) - log.info("created bundle with spec \(spec)") - var cgroupsPath: String if let cgPath = spec.linux?.cgroupsPath { cgroupsPath = cgPath @@ -52,30 +46,43 @@ actor ManagedContainer { cgroupsPath = "/container/\(id)" } - let cgManager = try Cgroup2Manager( - path: URL(filePath: cgroupsPath), + let bundle = try ContainerizationOCI.Bundle.create( + path: Self.craftBundlePath(id: id), + spec: spec + ) + log.info("created bundle with spec \(spec)") + + let cgManager = Cgroup2Manager( + group: URL(filePath: cgroupsPath), logger: log ) - try cgManager.toggleSubtreeControllers( - controllers: [.cpu, .cpuset, .hugetlb, .io, .memory, .pids], - enable: true - ) + try cgManager.create() - let initProcess = try ManagedProcess( - id: id, - stdio: stdio, - bundle: bundle, - cgroupManager: cgManager, - owningPid: nil, - log: log - ) - log.info("created managed init process") + do { + try cgManager.toggleSubtreeControllers( + controllers: [.cpu, .cpuset, .hugetlb, .io, .memory, .pids], + enable: true + ) - self.initProcess = initProcess - self.id = id - self.cgroupManager = cgManager - self.bundle = bundle - self.log = log + let initProcess = try ManagedProcess( + id: id, + stdio: stdio, + bundle: bundle, + cgroupManager: cgManager, + owningPid: nil, + log: log + ) + log.info("created managed init process") + + self.cgroupManager = cgManager + self.initProcess = initProcess + self.id = id + self.bundle = bundle + self.log = log + } catch { + try? cgManager.delete() + throw error + } } } @@ -104,7 +111,6 @@ extension ManagedContainer { id: id, stdio: stdio, bundle: self.bundle, - cgroupManager: self.cgroupManager, owningPid: self.initProcess.pid, log: self.log ) diff --git a/vminitd/Sources/vminitd/ManagedProcess.swift b/vminitd/Sources/vminitd/ManagedProcess.swift index ebec8736..b9200cf2 100644 --- a/vminitd/Sources/vminitd/ManagedProcess.swift +++ b/vminitd/Sources/vminitd/ManagedProcess.swift @@ -34,7 +34,7 @@ final class ManagedProcess: Sendable { private let syncPipe: FileHandle private let terminal: Bool private let bundle: ContainerizationOCI.Bundle - private let cgroupManager: Cgroup2Manager + private let cgroupManager: Cgroup2Manager? private struct State { init(io: IO) { @@ -75,7 +75,7 @@ final class ManagedProcess: Sendable { id: String, stdio: HostStdio, bundle: ContainerizationOCI.Bundle, - cgroupManager: Cgroup2Manager, + cgroupManager: Cgroup2Manager? = nil, owningPid: Int32? = nil, log: Logger ) throws { @@ -84,7 +84,6 @@ final class ManagedProcess: Sendable { Self.localizeLogger(log: &log, id: id) self.log = log self.owningPid = owningPid - self.cgroupManager = cgroupManager let syncPipe = Pipe() try syncPipe.setCloexec() @@ -138,6 +137,7 @@ final class ManagedProcess: Sendable { // Setup IO early. We expect the host to be listening already. try io.start(process: &process) + self.cgroupManager = cgroupManager self.process = process self.terminal = stdio.terminal self.bundle = bundle @@ -184,8 +184,17 @@ extension ManagedProcess { ]) $0.pid = pid - // First add to our cg, then ack the pid. - try self.cgroupManager.addProcess(pid: pid) + // Add to our cgroup. For execs (owningPid is non-nil) we'll + // see where the init process is actually located now (systemd + // loves to move all its processes to a child /init.scope cg). + if let cgroupManager { + try cgroupManager.addProcess(pid: pid) + } else { + if let owningPid { + let cgManager = try Cgroup2Manager.loadFromPid(pid: owningPid) + try cgManager.addProcess(pid: pid) + } + } log.info( "sending pid acknowledgement", diff --git a/vminitd/Sources/vminitd/Server+GRPC.swift b/vminitd/Sources/vminitd/Server+GRPC.swift index 9daf23e5..b9e1968c 100644 --- a/vminitd/Sources/vminitd/Server+GRPC.swift +++ b/vminitd/Sources/vminitd/Server+GRPC.swift @@ -439,7 +439,7 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContextAsyncProvid from: request.configuration ) - try ociAlterations(ociSpec: &ociSpec) + try ociAlterations(id: request.id, ociSpec: &ociSpec) guard let process = ociSpec.process else { throw ContainerizationError( @@ -940,7 +940,7 @@ extension Com_Apple_Containerization_Sandbox_V3_ConfigureHostsRequest { } extension Initd { - func ociAlterations(ociSpec: inout ContainerizationOCI.Spec) throws { + func ociAlterations(id: String, ociSpec: inout ContainerizationOCI.Spec) throws { guard var process = ociSpec.process else { throw ContainerizationError( .invalidArgument, @@ -954,6 +954,10 @@ extension Initd { ) } + if ociSpec.linux!.cgroupsPath.isEmpty { + ociSpec.linux!.cgroupsPath = "/container/\(id)" + } + if process.cwd.isEmpty { process.cwd = "/" }