Cgroup2Manager: Various adjustments (#270)

Systemd loves to move the root cgroups processes to one it created named
/init.scope and then write the root cgs subtree_control file. Because of
this we can't just add exec processes to the cg we made for the
container anymore as we'll get EBUSY. We should follow where the init
processes cg is actually at (/proc/pid/cgroup) and add it there.

vminitd/Sources/vmexec/ExecCommand.swift

@@ -46,14 +46,8 @@ struct ExecCommand: ParsableCommand {
         try execInNamespaces(process: process, log: log)
     }
 
-    static func enterNS(path: String, nsType: Int32) throws {
-        let fd = open(path, O_RDONLY)
-        if fd <= 0 {
-            throw App.Errno(stage: "open(ns)")
-        }
-        defer { close(fd) }
-
-        guard setns(fd, nsType) == 0 else {
+    static func enterNS(pidFd: Int32, nsType: Int32) throws {
+        guard setns(pidFd, nsType) == 0 else {
             throw App.Errno(stage: "setns(fd)")
         }
     }
@@ -65,10 +59,14 @@ struct ExecCommand: ParsableCommand {
         let syncPipe = FileHandle(fileDescriptor: 3)
         let ackPipe = FileHandle(fileDescriptor: 4)
 
-        try Self.enterNS(path: "/proc/\(self.parentPid)/ns/cgroup", nsType: CLONE_NEWCGROUP)
-        try Self.enterNS(path: "/proc/\(self.parentPid)/ns/pid", nsType: CLONE_NEWPID)
-        try Self.enterNS(path: "/proc/\(self.parentPid)/ns/uts", nsType: CLONE_NEWUTS)
-        try Self.enterNS(path: "/proc/\(self.parentPid)/ns/mnt", nsType: CLONE_NEWNS)
+        let pidFd = CZ_pidfd_open(Int32(parentPid), 0)
+        guard pidFd > 0 else {
+            throw App.Errno(stage: "pidfd_open(\(parentPid))")
+        }
+        try Self.enterNS(
+            pidFd: pidFd,
+            nsType: CLONE_NEWCGROUP | CLONE_NEWPID | CLONE_NEWUTS | CLONE_NEWNS
+        )
 
         let processID = fork()
 

vminitd/Sources/vminitd/Cgroup2Manager.swift

@@ -14,10 +14,17 @@
 // limitations under the License.
 //===----------------------------------------------------------------------===//
 
+#if os(Linux)
+
+#if canImport(Musl)
+import Musl
+#elseif canImport(Glibc)
+import Glibc
+#endif
+
 import ContainerizationOS
 import Foundation
 import Logging
-import Musl
 
 enum Cgroup2Controller: String {
     case pids
@@ -30,27 +37,71 @@ enum Cgroup2Controller: String {
 
 // Extremely simple cgroup manager. Our needs are simple for now, and this is
 // reflected in the type.
-internal struct Cgroup2Manager {
+struct Cgroup2Manager: Sendable {
     static let defaultMountPoint = URL(filePath: "/sys/fs/cgroup")
 
-    static let killFile = "cgroup.kill"
-    static let procsFile = "cgroup.procs"
-    static let subtreeControlFile = "cgroup.subtree_control"
+    private static let killFile = "cgroup.kill"
+    private static let procsFile = "cgroup.procs"
+    private static let subtreeControlFile = "cgroup.subtree_control"
+
+    private static let cg2Magic = 0x6367_7270
 
     private let mountPoint: URL
     private let path: URL
     private let logger: Logger?
 
     init(
-        mountPoint: URL = defaultMountPoint,
-        path: URL,
-        perms: Int16 = 0o755,
+        mountPoint: URL = Self.defaultMountPoint,
+        group: URL,
         logger: Logger? = nil
-    ) throws {
+    ) {
         self.mountPoint = mountPoint
-        self.path = mountPoint.appending(path: path.path)
+        self.path = mountPoint.appending(path: group.path)
         self.logger = logger
+    }
 
+    static func load(
+        mountPoint: URL = Self.defaultMountPoint,
+        group: URL,
+        logger: Logger? = nil
+    ) throws -> Cgroup2Manager {
+        let path = mountPoint.appending(path: group.path)
+        var s = statfs()
+        let res = statfs(path.path, &s)
+        if res != 0 {
+            throw Error.errno(errno: errno, message: "failed to statfs \(path.path)")
+        }
+        if Int64(s.f_type) != Self.cg2Magic {
+            throw Error.notCgroup
+        }
+        return Cgroup2Manager(
+            mountPoint: mountPoint,
+            group: group,
+            logger: logger
+        )
+    }
+
+    static func loadFromPid(pid: Int32, logger: Logger? = nil) throws -> Cgroup2Manager {
+        let procCgPath = URL(filePath: "/proc/\(pid)/cgroup")
+        let fh = try FileHandle(forReadingFrom: procCgPath)
+        guard let data = try fh.readToEnd() else {
+            throw Error.errno(errno: errno, message: "failed to read \(procCgPath)")
+        }
+
+        // If this fails we have bigger problems.
+        let str = String(data: data, encoding: .utf8)!
+        let parts = str.split(separator: ":")
+        if parts[0] != "0" {
+            throw Error.cgroup1
+        }
+
+        // We should really read /proc/pid/mountinfo, but for now just assume
+        // it's always at /sys/fs/cgroup.
+        let path = parts[1].trimmingCharacters(in: .whitespacesAndNewlines)
+        return Cgroup2Manager(group: URL(filePath: String(path)), logger: logger)
+    }
+
+    func create(perms: Int16 = 0o755) throws {
         self.logger?.info(
             "creating cgroup manager",
             metadata: [
@@ -110,6 +161,13 @@ internal struct Cgroup2Manager {
     }
 
     func addProcess(pid: Int32) throws {
+        self.logger?.debug(
+            "adding new proc to cgroup",
+            metadata: [
+                "mountpoint": "\(self.mountPoint.path)",
+                "path": "\(self.path.path)",
+            ])
+
         let pidStr = String(pid)
         try Self.writeValue(
             path: self.path,
@@ -143,13 +201,24 @@ internal struct Cgroup2Manager {
 
 extension Cgroup2Manager {
     enum Error: Swift.Error, CustomStringConvertible {
+        case notCgroup
+        case cgroup1
         case errno(errno: Int32, message: String)
+        case notExist(path: String)
 
         var description: String {
             switch self {
             case .errno(let errno, let message):
                 return "failed with errno \(errno): \(message)"
+            case .notExist(let path):
+                return "cgroup at path \(path) does not exist"
+            case .cgroup1:
+                return "tried to load a cgroup v1 path"
+            case .notCgroup:
+                return "path is not a cgroup mountpoint"
             }
         }
     }
 }
+
+#endif

vminitd/Sources/vminitd/ManagedContainer.swift

@@ -39,12 +39,6 @@ actor ManagedContainer {
         spec: ContainerizationOCI.Spec,
         log: Logger
     ) throws {
-        let bundle = try ContainerizationOCI.Bundle.create(
-            path: Self.craftBundlePath(id: id),
-            spec: spec
-        )
-        log.info("created bundle with spec \(spec)")
-
         var cgroupsPath: String
         if let cgPath = spec.linux?.cgroupsPath {
             cgroupsPath = cgPath
@@ -52,30 +46,43 @@ actor ManagedContainer {
             cgroupsPath = "/container/\(id)"
         }
 
-        let cgManager = try Cgroup2Manager(
-            path: URL(filePath: cgroupsPath),
+        let bundle = try ContainerizationOCI.Bundle.create(
+            path: Self.craftBundlePath(id: id),
+            spec: spec
+        )
+        log.info("created bundle with spec \(spec)")
+
+        let cgManager = Cgroup2Manager(
+            group: URL(filePath: cgroupsPath),
             logger: log
         )
-        try cgManager.toggleSubtreeControllers(
-            controllers: [.cpu, .cpuset, .hugetlb, .io, .memory, .pids],
-            enable: true
-        )
+        try cgManager.create()
 
-        let initProcess = try ManagedProcess(
-            id: id,
-            stdio: stdio,
-            bundle: bundle,
-            cgroupManager: cgManager,
-            owningPid: nil,
-            log: log
-        )
-        log.info("created managed init process")
+        do {
+            try cgManager.toggleSubtreeControllers(
+                controllers: [.cpu, .cpuset, .hugetlb, .io, .memory, .pids],
+                enable: true
+            )
 
-        self.initProcess = initProcess
-        self.id = id
-        self.cgroupManager = cgManager
-        self.bundle = bundle
-        self.log = log
+            let initProcess = try ManagedProcess(
+                id: id,
+                stdio: stdio,
+                bundle: bundle,
+                cgroupManager: cgManager,
+                owningPid: nil,
+                log: log
+            )
+            log.info("created managed init process")
+
+            self.cgroupManager = cgManager
+            self.initProcess = initProcess
+            self.id = id
+            self.bundle = bundle
+            self.log = log
+        } catch {
+            try? cgManager.delete()
+            throw error
+        }
     }
 }
 
@@ -104,7 +111,6 @@ extension ManagedContainer {
             id: id,
             stdio: stdio,
             bundle: self.bundle,
-            cgroupManager: self.cgroupManager,
             owningPid: self.initProcess.pid,
             log: self.log
         )

vminitd/Sources/vminitd/ManagedProcess.swift

@@ -34,7 +34,7 @@ final class ManagedProcess: Sendable {
     private let syncPipe: FileHandle
     private let terminal: Bool
     private let bundle: ContainerizationOCI.Bundle
-    private let cgroupManager: Cgroup2Manager
+    private let cgroupManager: Cgroup2Manager?
 
     private struct State {
         init(io: IO) {
@@ -75,7 +75,7 @@ final class ManagedProcess: Sendable {
         id: String,
         stdio: HostStdio,
         bundle: ContainerizationOCI.Bundle,
-        cgroupManager: Cgroup2Manager,
+        cgroupManager: Cgroup2Manager? = nil,
         owningPid: Int32? = nil,
         log: Logger
     ) throws {
@@ -84,7 +84,6 @@ final class ManagedProcess: Sendable {
         Self.localizeLogger(log: &log, id: id)
         self.log = log
         self.owningPid = owningPid
-        self.cgroupManager = cgroupManager
 
         let syncPipe = Pipe()
         try syncPipe.setCloexec()
@@ -138,6 +137,7 @@ final class ManagedProcess: Sendable {
         // Setup IO early. We expect the host to be listening already.
         try io.start(process: &process)
 
+        self.cgroupManager = cgroupManager
         self.process = process
         self.terminal = stdio.terminal
         self.bundle = bundle
@@ -184,8 +184,17 @@ extension ManagedProcess {
                 ])
             $0.pid = pid
 
-            // First add to our cg, then ack the pid.
-            try self.cgroupManager.addProcess(pid: pid)
+            // Add to our cgroup. For execs (owningPid is non-nil) we'll
+            // see where the init process is actually located now (systemd
+            // loves to move all its processes to a child /init.scope cg).
+            if let cgroupManager {
+                try cgroupManager.addProcess(pid: pid)
+            } else {
+                if let owningPid {
+                    let cgManager = try Cgroup2Manager.loadFromPid(pid: owningPid)
+                    try cgManager.addProcess(pid: pid)
+                }
+            }
 
             log.info(
                 "sending pid acknowledgement",

vminitd/Sources/vminitd/Server+GRPC.swift

@@ -439,7 +439,7 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContextAsyncProvid
                 from: request.configuration
             )
 
-            try ociAlterations(ociSpec: &ociSpec)
+            try ociAlterations(id: request.id, ociSpec: &ociSpec)
 
             guard let process = ociSpec.process else {
                 throw ContainerizationError(
@@ -940,7 +940,7 @@ extension Com_Apple_Containerization_Sandbox_V3_ConfigureHostsRequest {
 }
 
 extension Initd {
-    func ociAlterations(ociSpec: inout ContainerizationOCI.Spec) throws {
+    func ociAlterations(id: String, ociSpec: inout ContainerizationOCI.Spec) throws {
         guard var process = ociSpec.process else {
             throw ContainerizationError(
                 .invalidArgument,
@@ -954,6 +954,10 @@ extension Initd {
             )
         }
 
+        if ociSpec.linux!.cgroupsPath.isEmpty {
+            ociSpec.linux!.cgroupsPath = "/container/\(id)"
+        }
+
         if process.cwd.isEmpty {
             process.cwd = "/"
         }