From 79e07b43ce789b1e14f631d0fd0fc6d444f06483 Mon Sep 17 00:00:00 2001 From: Danny Canter Date: Wed, 13 Aug 2025 12:58:49 -0400 Subject: [PATCH] vminitd: Add init and execs to cgroup (#265) We currently weren't doing any cgroup setup whatsoever. For the most part this doesn't matter too much, however certain images that may fool around with cgroups don't like this :). Lets do the bare minimum these expect which is to at least have the process running in a nested cg and not apart of the root cg. --- Sources/Containerization/Agent/Vminitd.swift | 2 +- Sources/Containerization/LinuxContainer.swift | 7 +- vminitd/Sources/vminitd/CgroupManager.swift | 148 ++++++++++++++++++ .../Sources/vminitd/ManagedContainer.swift | 51 ++++-- vminitd/Sources/vminitd/ManagedProcess.swift | 7 +- 5 files changed, 194 insertions(+), 21 deletions(-) create mode 100644 vminitd/Sources/vminitd/CgroupManager.swift diff --git a/Sources/Containerization/Agent/Vminitd.swift b/Sources/Containerization/Agent/Vminitd.swift index bfa005cb..20c2fed2 100644 --- a/Sources/Containerization/Agent/Vminitd.swift +++ b/Sources/Containerization/Agent/Vminitd.swift @@ -66,7 +66,7 @@ extension Vminitd: VirtualMachineAgent { } // Setup root cg subtree_control. - let data = "+memory +pids +io +cpu +cpuset".data(using: .utf8)! + let data = "+memory +pids +io +cpu +cpuset +hugetlb".data(using: .utf8)! try await writeFile( path: "/sys/fs/cgroup/cgroup.subtree_control", data: data, diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index a149d106..2728a6ed 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -421,7 +421,8 @@ public final class LinuxContainer: Container, Sendable { readonly: false ), linux: .init( - resources: .init() + resources: .init(), + cgroupsPath: "/container/\(id)" ) ) } @@ -436,9 +437,7 @@ public final class LinuxContainer: Container, Sendable { spec.hostname = config.hostname // Linux toggles. - var linux = ContainerizationOCI.Linux.init() - linux.sysctl = config.sysctl - spec.linux = linux + spec.linux?.sysctl = config.sysctl return spec } diff --git a/vminitd/Sources/vminitd/CgroupManager.swift b/vminitd/Sources/vminitd/CgroupManager.swift new file mode 100644 index 00000000..17a93400 --- /dev/null +++ b/vminitd/Sources/vminitd/CgroupManager.swift @@ -0,0 +1,148 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the Containerization project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOS +import Foundation +import Logging +import Musl + +enum CgroupController: String { + case pids + case memory + case cpuset + case cpu + case io + case hugetlb +} + +// Extremely simple cgroup manager. Our needs are simple for now, and this is +// reflected in the type. +internal struct CgroupManager { + static let defaultMountPoint = URL(filePath: "/sys/fs/cgroup") + + static let killFile = "cgroup.kill" + static let procsFile = "cgroup.procs" + static let subtreeControlFile = "cgroup.subtree_control" + + private let mountPoint: URL + private let path: URL + private let logger: Logger? + + init( + mountPoint: URL = defaultMountPoint, + path: URL, + perms: Int16 = 0o755, + logger: Logger? = nil + ) throws { + self.mountPoint = mountPoint + self.path = mountPoint.appending(path: path.path) + self.logger = logger + + self.logger?.error( + "creating cgroup manager", + metadata: [ + "mountpoint": "\(self.mountPoint.path)", + "path": "\(self.path.path)", + ]) + + try FileManager.default.createDirectory( + at: self.path, + withIntermediateDirectories: true, + attributes: [.posixPermissions: perms] + ) + } + + private static func writeValue(path: URL, value: String, fileName: String) throws { + let file = path.appending(path: fileName) + let fd = open(file.path, O_WRONLY, 0) + if fd == -1 { + throw Error.errno(errno: errno, message: "failed to open \(file.path)") + } + defer { Musl.close(fd) } + + let bytes = Array(value.utf8) + let res = Syscall.retrying { + bytes.withUnsafeBytes { write(fd, $0.baseAddress!, bytes.count) } + } + if res == -1 { + throw Error.errno(errno: errno, message: "failed to write to \(file.path)") + } + } + + func toggleSubtreeControllers(controllers: [CgroupController], enable: Bool) throws { + let value = controllers.map { (enable ? "+" : "-") + $0.rawValue }.joined(separator: " ") + let mountComponents = self.mountPoint.pathComponents + let pathComponents = self.path.pathComponents + + // First ensure it's set on the root. + var current = self.mountPoint + try Self.writeValue( + path: current, + value: value, + fileName: Self.subtreeControlFile + ) + + // Toggle everything except the leaf, as otherwise we won't be able to write + // to cgroup.procs, and what fun is that :) + if mountComponents.count < pathComponents.count - 1 { + for i in mountComponents.count...pathComponents.count - 2 { + current = current.appending(path: pathComponents[i]) + try Self.writeValue( + path: current, + value: value, + fileName: Self.subtreeControlFile + ) + } + } + } + + func addProcess(pid: Int32) throws { + let pidStr = String(pid) + try Self.writeValue( + path: self.path, + value: pidStr, + fileName: Self.procsFile + ) + } + + func kill() throws { + try Self.writeValue( + path: self.path, + value: "1", + fileName: Self.killFile + ) + } + + func delete(force: Bool = false) throws { + if force { + try self.kill() + } + try FileManager.default.removeItem(at: self.path) + } +} + +extension CgroupManager { + enum Error: Swift.Error, CustomStringConvertible { + case errno(errno: Int32, message: String) + + var description: String { + switch self { + case .errno(let errno, let message): + return "failed with errno \(errno): \(message)" + } + } + } +} diff --git a/vminitd/Sources/vminitd/ManagedContainer.swift b/vminitd/Sources/vminitd/ManagedContainer.swift index ce6ade9c..293145c0 100644 --- a/vminitd/Sources/vminitd/ManagedContainer.swift +++ b/vminitd/Sources/vminitd/ManagedContainer.swift @@ -24,9 +24,10 @@ actor ManagedContainer { let id: String let initProcess: ManagedProcess - private let _log: Logger - private let _bundle: ContainerizationOCI.Bundle - private var _execs: [String: ManagedProcess] = [:] + private let cgroupManager: CgroupManager + private let log: Logger + private let bundle: ContainerizationOCI.Bundle + private var execs: [String: ManagedProcess] = [:] var pid: Int32 { self.initProcess.pid @@ -44,10 +45,27 @@ actor ManagedContainer { ) log.info("created bundle with spec \(spec)") + var cgroupsPath: String + if let cgPath = spec.linux?.cgroupsPath { + cgroupsPath = cgPath + } else { + cgroupsPath = "/container/\(id)" + } + + let cgManager = try CgroupManager( + path: URL(filePath: cgroupsPath), + logger: log + ) + try cgManager.toggleSubtreeControllers( + controllers: [.cpu, .cpuset, .hugetlb, .io, .memory, .pids], + enable: true + ) + let initProcess = try ManagedProcess( id: id, stdio: stdio, bundle: bundle, + cgroupManager: cgManager, owningPid: nil, log: log ) @@ -55,14 +73,15 @@ actor ManagedContainer { self.initProcess = initProcess self.id = id - self._bundle = bundle - self._log = log + self.cgroupManager = cgManager + self.bundle = bundle + self.log = log } } extension ManagedContainer { private func ensureExecExists(_ id: String) throws { - if self._execs[id] == nil { + if self.execs[id] == nil { throw ContainerizationError( .invalidState, message: "exec \(id) does not exist in container \(self.id)" @@ -77,18 +96,19 @@ extension ManagedContainer { ) throws { // Write the process config to the bundle, and pass this on // over to ManagedProcess to deal with. - try self._bundle.createExecSpec( + try self.bundle.createExecSpec( id: id, process: process ) let process = try ManagedProcess( id: id, stdio: stdio, - bundle: self._bundle, + bundle: self.bundle, + cgroupManager: self.cgroupManager, owningPid: self.initProcess.pid, - log: self._log + log: self.log ) - self._execs[id] = process + self.execs[id] = process } func start(execID: String) async throws -> Int32 { @@ -119,22 +139,23 @@ extension ManagedContainer { func deleteExec(id: String) throws { try ensureExecExists(id) do { - try self._bundle.deleteExecSpec(id: id) + try self.bundle.deleteExecSpec(id: id) } catch { - self._log.error("failed to remove exec spec from filesystem: \(error)") + self.log.error("failed to remove exec spec from filesystem: \(error)") } - self._execs.removeValue(forKey: id) + self.execs.removeValue(forKey: id) } func delete() throws { - try self._bundle.delete() + try self.bundle.delete() + try self.cgroupManager.delete(force: true) } func getExecOrInit(execID: String) throws -> ManagedProcess { if execID == self.id { return self.initProcess } - guard let proc = self._execs[execID] else { + guard let proc = self.execs[execID] else { throw ContainerizationError( .invalidState, message: "exec \(execID) does not exist in container \(self.id)" diff --git a/vminitd/Sources/vminitd/ManagedProcess.swift b/vminitd/Sources/vminitd/ManagedProcess.swift index 05734bed..dbe1b53b 100644 --- a/vminitd/Sources/vminitd/ManagedProcess.swift +++ b/vminitd/Sources/vminitd/ManagedProcess.swift @@ -34,6 +34,7 @@ final class ManagedProcess: Sendable { private let syncPipe: FileHandle private let terminal: Bool private let bundle: ContainerizationOCI.Bundle + private let cgroupManager: CgroupManager private struct State { init(io: IO) { @@ -74,6 +75,7 @@ final class ManagedProcess: Sendable { id: String, stdio: HostStdio, bundle: ContainerizationOCI.Bundle, + cgroupManager: CgroupManager, owningPid: Int32? = nil, log: Logger ) throws { @@ -82,6 +84,7 @@ final class ManagedProcess: Sendable { Self.localizeLogger(log: &log, id: id) self.log = log self.owningPid = owningPid + self.cgroupManager = cgroupManager let syncPipe = Pipe() try syncPipe.setCloexec() @@ -181,7 +184,9 @@ extension ManagedProcess { ]) $0.pid = pid - // Ack the pid from the child. + // First add to our cg, then ack the pid. + try self.cgroupManager.addProcess(pid: pid) + log.info( "sending pid acknowledgement", metadata: [