diff --git a/vminitd/Sources/vmexec/ExecCommand.swift b/vminitd/Sources/vmexec/ExecCommand.swift index e26e4b86..501e118c 100644 --- a/vminitd/Sources/vmexec/ExecCommand.swift +++ b/vminitd/Sources/vmexec/ExecCommand.swift @@ -46,14 +46,8 @@ struct ExecCommand: ParsableCommand { try execInNamespaces(process: process, log: log) } - static func enterNS(path: String, nsType: Int32) throws { - let fd = open(path, O_RDONLY) - if fd <= 0 { - throw App.Errno(stage: "open(ns)") - } - defer { close(fd) } - - guard setns(fd, nsType) == 0 else { + static func enterNS(pidFd: Int32, nsType: Int32) throws { + guard setns(pidFd, nsType) == 0 else { throw App.Errno(stage: "setns(fd)") } } @@ -65,10 +59,14 @@ struct ExecCommand: ParsableCommand { let syncPipe = FileHandle(fileDescriptor: 3) let ackPipe = FileHandle(fileDescriptor: 4) - try Self.enterNS(path: "/proc/\(self.parentPid)/ns/cgroup", nsType: CLONE_NEWCGROUP) - try Self.enterNS(path: "/proc/\(self.parentPid)/ns/pid", nsType: CLONE_NEWPID) - try Self.enterNS(path: "/proc/\(self.parentPid)/ns/uts", nsType: CLONE_NEWUTS) - try Self.enterNS(path: "/proc/\(self.parentPid)/ns/mnt", nsType: CLONE_NEWNS) + let pidFd = CZ_pidfd_open(Int32(parentPid), 0) + guard pidFd > 0 else { + throw App.Errno(stage: "pidfd_open(\(parentPid))") + } + try Self.enterNS( + pidFd: pidFd, + nsType: CLONE_NEWCGROUP | CLONE_NEWPID | CLONE_NEWUTS | CLONE_NEWNS + ) let processID = fork() diff --git a/vminitd/Sources/vminitd/CgroupManager.swift b/vminitd/Sources/vminitd/Cgroup2Manager.swift similarity index 63% rename from vminitd/Sources/vminitd/CgroupManager.swift rename to vminitd/Sources/vminitd/Cgroup2Manager.swift index de745d11..ae02a1c3 100644 --- a/vminitd/Sources/vminitd/CgroupManager.swift +++ b/vminitd/Sources/vminitd/Cgroup2Manager.swift @@ -14,10 +14,17 @@ // limitations under the License. //===----------------------------------------------------------------------===// +#if os(Linux) + +#if canImport(Musl) +import Musl +#elseif canImport(Glibc) +import Glibc +#endif + import ContainerizationOS import Foundation import Logging -import Musl enum Cgroup2Controller: String { case pids @@ -30,27 +37,71 @@ enum Cgroup2Controller: String { // Extremely simple cgroup manager. Our needs are simple for now, and this is // reflected in the type. -internal struct Cgroup2Manager { +struct Cgroup2Manager: Sendable { static let defaultMountPoint = URL(filePath: "/sys/fs/cgroup") - static let killFile = "cgroup.kill" - static let procsFile = "cgroup.procs" - static let subtreeControlFile = "cgroup.subtree_control" + private static let killFile = "cgroup.kill" + private static let procsFile = "cgroup.procs" + private static let subtreeControlFile = "cgroup.subtree_control" + + private static let cg2Magic = 0x6367_7270 private let mountPoint: URL private let path: URL private let logger: Logger? init( - mountPoint: URL = defaultMountPoint, - path: URL, - perms: Int16 = 0o755, + mountPoint: URL = Self.defaultMountPoint, + group: URL, logger: Logger? = nil - ) throws { + ) { self.mountPoint = mountPoint - self.path = mountPoint.appending(path: path.path) + self.path = mountPoint.appending(path: group.path) self.logger = logger + } + static func load( + mountPoint: URL = Self.defaultMountPoint, + group: URL, + logger: Logger? = nil + ) throws -> Cgroup2Manager { + let path = mountPoint.appending(path: group.path) + var s = statfs() + let res = statfs(path.path, &s) + if res != 0 { + throw Error.errno(errno: errno, message: "failed to statfs \(path.path)") + } + if Int64(s.f_type) != Self.cg2Magic { + throw Error.notCgroup + } + return Cgroup2Manager( + mountPoint: mountPoint, + group: group, + logger: logger + ) + } + + static func loadFromPid(pid: Int32, logger: Logger? = nil) throws -> Cgroup2Manager { + let procCgPath = URL(filePath: "/proc/\(pid)/cgroup") + let fh = try FileHandle(forReadingFrom: procCgPath) + guard let data = try fh.readToEnd() else { + throw Error.errno(errno: errno, message: "failed to read \(procCgPath)") + } + + // If this fails we have bigger problems. + let str = String(data: data, encoding: .utf8)! + let parts = str.split(separator: ":") + if parts[0] != "0" { + throw Error.cgroup1 + } + + // We should really read /proc/pid/mountinfo, but for now just assume + // it's always at /sys/fs/cgroup. + let path = parts[1].trimmingCharacters(in: .whitespacesAndNewlines) + return Cgroup2Manager(group: URL(filePath: String(path)), logger: logger) + } + + func create(perms: Int16 = 0o755) throws { self.logger?.info( "creating cgroup manager", metadata: [ @@ -110,6 +161,13 @@ internal struct Cgroup2Manager { } func addProcess(pid: Int32) throws { + self.logger?.debug( + "adding new proc to cgroup", + metadata: [ + "mountpoint": "\(self.mountPoint.path)", + "path": "\(self.path.path)", + ]) + let pidStr = String(pid) try Self.writeValue( path: self.path, @@ -143,13 +201,24 @@ internal struct Cgroup2Manager { extension Cgroup2Manager { enum Error: Swift.Error, CustomStringConvertible { + case notCgroup + case cgroup1 case errno(errno: Int32, message: String) + case notExist(path: String) var description: String { switch self { case .errno(let errno, let message): return "failed with errno \(errno): \(message)" + case .notExist(let path): + return "cgroup at path \(path) does not exist" + case .cgroup1: + return "tried to load a cgroup v1 path" + case .notCgroup: + return "path is not a cgroup mountpoint" } } } } + +#endif diff --git a/vminitd/Sources/vminitd/ManagedContainer.swift b/vminitd/Sources/vminitd/ManagedContainer.swift index ed15a2ae..29b9ce4e 100644 --- a/vminitd/Sources/vminitd/ManagedContainer.swift +++ b/vminitd/Sources/vminitd/ManagedContainer.swift @@ -39,12 +39,6 @@ actor ManagedContainer { spec: ContainerizationOCI.Spec, log: Logger ) throws { - let bundle = try ContainerizationOCI.Bundle.create( - path: Self.craftBundlePath(id: id), - spec: spec - ) - log.info("created bundle with spec \(spec)") - var cgroupsPath: String if let cgPath = spec.linux?.cgroupsPath { cgroupsPath = cgPath @@ -52,30 +46,43 @@ actor ManagedContainer { cgroupsPath = "/container/\(id)" } - let cgManager = try Cgroup2Manager( - path: URL(filePath: cgroupsPath), + let bundle = try ContainerizationOCI.Bundle.create( + path: Self.craftBundlePath(id: id), + spec: spec + ) + log.info("created bundle with spec \(spec)") + + let cgManager = Cgroup2Manager( + group: URL(filePath: cgroupsPath), logger: log ) - try cgManager.toggleSubtreeControllers( - controllers: [.cpu, .cpuset, .hugetlb, .io, .memory, .pids], - enable: true - ) + try cgManager.create() - let initProcess = try ManagedProcess( - id: id, - stdio: stdio, - bundle: bundle, - cgroupManager: cgManager, - owningPid: nil, - log: log - ) - log.info("created managed init process") + do { + try cgManager.toggleSubtreeControllers( + controllers: [.cpu, .cpuset, .hugetlb, .io, .memory, .pids], + enable: true + ) - self.initProcess = initProcess - self.id = id - self.cgroupManager = cgManager - self.bundle = bundle - self.log = log + let initProcess = try ManagedProcess( + id: id, + stdio: stdio, + bundle: bundle, + cgroupManager: cgManager, + owningPid: nil, + log: log + ) + log.info("created managed init process") + + self.cgroupManager = cgManager + self.initProcess = initProcess + self.id = id + self.bundle = bundle + self.log = log + } catch { + try? cgManager.delete() + throw error + } } } @@ -104,7 +111,6 @@ extension ManagedContainer { id: id, stdio: stdio, bundle: self.bundle, - cgroupManager: self.cgroupManager, owningPid: self.initProcess.pid, log: self.log ) diff --git a/vminitd/Sources/vminitd/ManagedProcess.swift b/vminitd/Sources/vminitd/ManagedProcess.swift index ebec8736..b9200cf2 100644 --- a/vminitd/Sources/vminitd/ManagedProcess.swift +++ b/vminitd/Sources/vminitd/ManagedProcess.swift @@ -34,7 +34,7 @@ final class ManagedProcess: Sendable { private let syncPipe: FileHandle private let terminal: Bool private let bundle: ContainerizationOCI.Bundle - private let cgroupManager: Cgroup2Manager + private let cgroupManager: Cgroup2Manager? private struct State { init(io: IO) { @@ -75,7 +75,7 @@ final class ManagedProcess: Sendable { id: String, stdio: HostStdio, bundle: ContainerizationOCI.Bundle, - cgroupManager: Cgroup2Manager, + cgroupManager: Cgroup2Manager? = nil, owningPid: Int32? = nil, log: Logger ) throws { @@ -84,7 +84,6 @@ final class ManagedProcess: Sendable { Self.localizeLogger(log: &log, id: id) self.log = log self.owningPid = owningPid - self.cgroupManager = cgroupManager let syncPipe = Pipe() try syncPipe.setCloexec() @@ -138,6 +137,7 @@ final class ManagedProcess: Sendable { // Setup IO early. We expect the host to be listening already. try io.start(process: &process) + self.cgroupManager = cgroupManager self.process = process self.terminal = stdio.terminal self.bundle = bundle @@ -184,8 +184,17 @@ extension ManagedProcess { ]) $0.pid = pid - // First add to our cg, then ack the pid. - try self.cgroupManager.addProcess(pid: pid) + // Add to our cgroup. For execs (owningPid is non-nil) we'll + // see where the init process is actually located now (systemd + // loves to move all its processes to a child /init.scope cg). + if let cgroupManager { + try cgroupManager.addProcess(pid: pid) + } else { + if let owningPid { + let cgManager = try Cgroup2Manager.loadFromPid(pid: owningPid) + try cgManager.addProcess(pid: pid) + } + } log.info( "sending pid acknowledgement", diff --git a/vminitd/Sources/vminitd/Server+GRPC.swift b/vminitd/Sources/vminitd/Server+GRPC.swift index 9daf23e5..b9e1968c 100644 --- a/vminitd/Sources/vminitd/Server+GRPC.swift +++ b/vminitd/Sources/vminitd/Server+GRPC.swift @@ -439,7 +439,7 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContextAsyncProvid from: request.configuration ) - try ociAlterations(ociSpec: &ociSpec) + try ociAlterations(id: request.id, ociSpec: &ociSpec) guard let process = ociSpec.process else { throw ContainerizationError( @@ -940,7 +940,7 @@ extension Com_Apple_Containerization_Sandbox_V3_ConfigureHostsRequest { } extension Initd { - func ociAlterations(ociSpec: inout ContainerizationOCI.Spec) throws { + func ociAlterations(id: String, ociSpec: inout ContainerizationOCI.Spec) throws { guard var process = ociSpec.process else { throw ContainerizationError( .invalidArgument, @@ -954,6 +954,10 @@ extension Initd { ) } + if ociSpec.linux!.cgroupsPath.isEmpty { + ociSpec.linux!.cgroupsPath = "/container/\(id)" + } + if process.cwd.isEmpty { process.cwd = "/" }