From 0a4ff1b737c37563196754f62a3218873a49c6a7 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 22 Jul 2025 14:53:52 -0400 Subject: [PATCH] Add ContainerManager (#200) A ContainerManager is a type that handles more of the required resource needed to create and run a container. --------- Signed-off-by: crosbymichael --- .../Containerization/ContainerManager.swift | 418 ++++++++++++++++++ .../Image/ImageStore/ImageStore.swift | 49 +- .../VZVirtualMachineManager.swift | 2 +- Sources/Integration/Suite.swift | 6 +- Sources/Integration/VMTests.swift | 41 ++ Sources/cctl/ContainerStore.swift | 135 ------ Sources/cctl/RunCommand.swift | 22 +- 7 files changed, 517 insertions(+), 156 deletions(-) create mode 100644 Sources/Containerization/ContainerManager.swift delete mode 100644 Sources/cctl/ContainerStore.swift diff --git a/Sources/Containerization/ContainerManager.swift b/Sources/Containerization/ContainerManager.swift new file mode 100644 index 00000000..b012d1fb --- /dev/null +++ b/Sources/Containerization/ContainerManager.swift @@ -0,0 +1,418 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the Containerization project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +#if os(macOS) + +import ContainerizationError +import ContainerizationOCI +import ContainerizationOS +import Foundation +import ContainerizationExtras +import Virtualization +import vmnet + +/// A manager for creating and running containers. +/// Supports container networking options. +public struct ContainerManager: Sendable { + public let imageStore: ImageStore + private let vmm: VirtualMachineManager + private let network: Network? + + private var containerRoot: URL { + self.imageStore.path.appendingPathComponent("containers") + } + + /// A network that can allocate and release interfaces for use with containers. + public protocol Network: Sendable { + func create(_ id: String) throws -> Interface? + func release(_ id: String) throws + } + + /// A network backed by vmnet on macOS. + @available(macOS 26.0, *) + public struct VmnetNetwork: Network { + private let allocator: Allocator + nonisolated(unsafe) private let reference: vmnet_network_ref + + /// The IPv4 subnet of this network. + public let subnet: CIDRAddress + + /// The gateway address of this network. + public var gateway: IPv4Address { + subnet.gateway + } + + struct Allocator: Sendable { + private let addressAllocator: any AddressAllocator + private let cidr: CIDRAddress + private var allocations: [String: UInt32] + + init(cidr: CIDRAddress) throws { + self.cidr = cidr + self.allocations = .init() + let size = Int(cidr.upper.value - cidr.lower.value - 3) + self.addressAllocator = try UInt32.rotatingAllocator( + lower: cidr.lower.value + 2, + size: UInt32(size) + ) + } + + func allocate(_ id: String) throws -> String { + if allocations[id] != nil { + throw ContainerizationError(.exists, message: "allocation with id \(id) already exists") + } + let index = try addressAllocator.allocate() + let ip = IPv4Address(fromValue: index) + return try CIDRAddress(ip, prefixLength: cidr.prefixLength).description + } + + func release(_ id: String) throws { + if let index = self.allocations[id] { + try addressAllocator.release(index) + } + } + } + + /// A network interface supporting the vmnet_network_ref. + public struct Interface: Containerization.Interface, VZInterface, Sendable { + public let address: String + public let gateway: String? + public let macAddress: String? + + nonisolated(unsafe) private let reference: vmnet_network_ref + + public init( + reference: vmnet_network_ref, + address: String, + gateway: String, + macAddress: String? = nil + ) { + self.address = address + self.gateway = gateway + self.macAddress = macAddress + self.reference = reference + } + + /// Returns the underlying `VZVirtioNetworkDeviceConfiguration`. + public func device() throws -> VZVirtioNetworkDeviceConfiguration { + let config = VZVirtioNetworkDeviceConfiguration() + if let macAddress = self.macAddress { + guard let mac = VZMACAddress(string: macAddress) else { + throw ContainerizationError(.invalidArgument, message: "invalid mac address \(macAddress)") + } + config.macAddress = mac + } + config.attachment = VZVmnetNetworkDeviceAttachment(network: self.reference) + return config + } + } + + /// Creates a new network. + /// - Parameter subnet: The subnet to use for this network. + public init(subnet: String? = nil) throws { + var status: vmnet_return_t = .VMNET_FAILURE + guard let config = vmnet_network_configuration_create(.VMNET_SHARED_MODE, &status) else { + throw ContainerizationError(.unsupported, message: "failed to create vmnet config with status \(status)") + } + + vmnet_network_configuration_disable_dhcp(config) + + if let subnet { + try Self.configureSubnet(config, subnet: try CIDRAddress(subnet)) + } + + guard let ref = vmnet_network_create(config, &status), status == .VMNET_SUCCESS else { + throw ContainerizationError(.unsupported, message: "failed to create vmnet network with status \(status)") + } + + let cidr = try Self.getSubnet(ref) + + self.allocator = try .init(cidr: cidr) + self.subnet = cidr + self.reference = ref + } + + /// Returns a new interface for use with a container. + /// - Parameter id: The container ID. + public func create(_ id: String) throws -> Containerization.Interface? { + let address = try allocator.allocate(id) + return Self.Interface( + reference: self.reference, + address: address, + gateway: self.gateway.description, + ) + } + + /// Performs cleanup of an interface. + /// - Parameter id: The container ID. + public func release(_ id: String) throws { + try allocator.release(id) + } + + private static func getSubnet(_ ref: vmnet_network_ref) throws -> CIDRAddress { + var subnet = in_addr() + var mask = in_addr() + vmnet_network_get_ipv4_subnet(ref, &subnet, &mask) + + let sa = UInt32(bigEndian: subnet.s_addr) + let mv = UInt32(bigEndian: mask.s_addr) + + let lower = IPv4Address(fromValue: sa & mv) + let upper = IPv4Address(fromValue: lower.value + ~mv) + + return try CIDRAddress(lower: lower, upper: upper) + } + + private static func configureSubnet(_ config: vmnet_network_configuration_ref, subnet: CIDRAddress) throws { + let gateway = subnet.gateway + + var ga = in_addr() + inet_pton(AF_INET, gateway.description, &ga) + + let mask = IPv4Address(fromValue: subnet.prefixLength.prefixMask32) + var ma = in_addr() + inet_pton(AF_INET, mask.description, &ma) + + guard vmnet_network_configuration_set_ipv4_subnet(config, &ga, &ma) == .VMNET_SUCCESS else { + throw ContainerizationError(.internalError, message: "failed to set subnet \(subnet) for network") + } + } + } + + /// Create a new manager with the provided kernel and initfs mount. + public init( + kernel: Kernel, + initfs: Mount, + network: Network? = nil + ) throws { + self.imageStore = ImageStore.default + self.network = network + try Self.createRootDirectory(path: self.imageStore.path) + self.vmm = VZVirtualMachineManager( + kernel: kernel, + initialFilesystem: initfs, + bootlog: self.imageStore.path.appendingPathComponent("bootlog.log").absolutePath() + ) + } + + /// Create a new manager with the provided kernel and image reference for the initfs. + public init( + kernel: Kernel, + initfsReference: String, + network: Network? = nil + ) async throws { + self.imageStore = ImageStore.default + self.network = network + try Self.createRootDirectory(path: self.imageStore.path) + + let initPath = self.imageStore.path.appendingPathComponent("initfs.ext4") + let initImage = try await self.imageStore.getInitImage(reference: initfsReference) + let initfs = try await { + do { + return try await initImage.initBlock(at: initPath, for: .linuxArm) + } catch let err as ContainerizationError { + guard err.code == .exists else { + throw err + } + return .block( + format: "ext4", + source: initPath.absolutePath(), + destination: "/", + options: ["ro"] + ) + } + }() + + self.vmm = VZVirtualMachineManager( + kernel: kernel, + initialFilesystem: initfs, + bootlog: self.imageStore.path.appendingPathComponent("bootlog.log").absolutePath() + ) + } + + /// Create a new manager with the provided vmm and network. + public init( + vmm: any VirtualMachineManager, + network: Network? = nil + ) throws { + self.imageStore = ImageStore.default + try Self.createRootDirectory(path: self.imageStore.path) + self.network = network + self.vmm = vmm + } + + private static func createRootDirectory(path: URL) throws { + try FileManager.default.createDirectory( + at: path.appendingPathComponent("containers"), + withIntermediateDirectories: true + ) + } + + /// Returns a new container from the provided image reference. + /// - Parameters: + /// - id: The container ID. + /// - reference: The image reference. + /// - rootfsSizeInBytes: The size of the root filesystem in bytes. Defaults to 8 GiB. + public func create( + _ id: String, + reference: String, + rootfsSizeInBytes: UInt64 = 8.gib(), + configuration: (inout LinuxContainer.Configuration) throws -> Void + ) async throws -> LinuxContainer { + let image = try await imageStore.get(reference: reference, pull: true) + return try await create( + id, + image: image, + rootfsSizeInBytes: rootfsSizeInBytes, + configuration: configuration + ) + } + + /// Returns a new container from the provided image. + /// - Parameters: + /// - id: The container ID. + /// - image: The image. + /// - rootfsSizeInBytes: The size of the root filesystem in bytes. Defaults to 8 GiB. + public func create( + _ id: String, + image: Image, + rootfsSizeInBytes: UInt64 = 8.gib(), + configuration: (inout LinuxContainer.Configuration) throws -> Void + ) async throws -> LinuxContainer { + let path = try createContainerRoot(id) + + let rootfs = try await unpack( + image: image, + destination: path.appendingPathComponent("rootfs.ext4"), + size: rootfsSizeInBytes + ) + return try await create( + id, + image: image, + rootfs: rootfs, + configuration: configuration + ) + } + + /// Returns a new container from the provided image and root filesystem mount. + /// - Parameters: + /// - id: The container ID. + /// - image: The image. + /// - rootfs: The root filesystem mount pointing to an existing block file. + public func create( + _ id: String, + image: Image, + rootfs: Mount, + configuration: (inout LinuxContainer.Configuration) throws -> Void + ) async throws -> LinuxContainer { + let imageConfig = try await image.config(for: .current).config + return try LinuxContainer( + id, + rootfs: rootfs, + vmm: self.vmm + ) { config in + if let imageConfig { + config.process = .init(from: imageConfig) + } + if let interface = try self.network?.create(id) { + config.interfaces = [interface] + config.dns = .init(nameservers: [interface.gateway!]) + } + try configuration(&config) + } + } + + /// Returns an existing container from the provided image and root filesystem mount. + /// - Parameters: + /// - id: The container ID. + /// - image: The image. + public func get( + _ id: String, + image: Image, + ) async throws -> LinuxContainer { + let path = containerRoot.appendingPathComponent(id) + guard FileManager.default.fileExists(atPath: path.absolutePath()) else { + throw ContainerizationError(.notFound, message: "\(id) does not exist") + } + + let rootfs: Mount = .block( + format: "ext4", + source: path.appendingPathComponent("rootfs.ext4").absolutePath(), + destination: "/", + options: [] + ) + + let imageConfig = try await image.config(for: .current).config + return try LinuxContainer( + id, + rootfs: rootfs, + vmm: self.vmm + ) { config in + if let imageConfig { + config.process = .init(from: imageConfig) + } + if let interface = try self.network?.create(id) { + config.interfaces = [interface] + config.dns = .init(nameservers: [interface.gateway!]) + } + } + } + + /// Performs the cleanup of a container. + /// - Parameter id: The container ID. + public func delete(_ id: String) throws { + try self.network?.release(id) + let path = containerRoot.appendingPathComponent(id) + try FileManager.default.removeItem(at: path) + } + + private func createContainerRoot(_ id: String) throws -> URL { + let path = containerRoot.appendingPathComponent(id) + try FileManager.default.createDirectory(at: path, withIntermediateDirectories: false) + return path + } + + private func unpack(image: Image, destination: URL, size: UInt64) async throws -> Mount { + do { + let unpacker = EXT4Unpacker(blockSizeInBytes: size) + return try await unpacker.unpack(image, for: .current, at: destination) + } catch let err as ContainerizationError { + if err.code == .exists { + return .block( + format: "ext4", + source: destination.absolutePath(), + destination: "/", + options: [] + ) + } + throw err + } + } +} + +extension CIDRAddress { + /// The gateway address of the network. + public var gateway: IPv4Address { + IPv4Address(fromValue: self.lower.value + 1) + } +} + +@available(macOS 26.0, *) +private struct SendableReference: Sendable { + nonisolated(unsafe) private let reference: vmnet_network_ref +} + +#endif diff --git a/Sources/Containerization/Image/ImageStore/ImageStore.swift b/Sources/Containerization/Image/ImageStore/ImageStore.swift index fa8b9eb9..d0ebcff7 100644 --- a/Sources/Containerization/Image/ImageStore/ImageStore.swift +++ b/Sources/Containerization/Image/ImageStore/ImageStore.swift @@ -22,16 +22,47 @@ import Foundation /// An ImageStore handles the mappings between an image's /// reference and the underlying descriptor inside of a content store. public actor ImageStore: Sendable { + /// The ImageStore path it was created with. + public nonisolated let path: URL + private let referenceManager: ReferenceManager internal let contentStore: ContentStore internal let lock: AsyncLock = AsyncLock() - public init(path: URL, contentStore: ContentStore) throws { + public init(path: URL, contentStore: ContentStore? = nil) throws { try FileManager.default.createDirectory(at: path, withIntermediateDirectories: true) - self.contentStore = contentStore + if let contentStore { + self.contentStore = contentStore + } else { + self.contentStore = try LocalContentStore(path: path.appendingPathComponent("content")) + } + + self.path = path self.referenceManager = try ReferenceManager(path: path) } + + /// Return the default image store for the current user. + public static let `default`: ImageStore = { + do { + let root = try defaultRoot() + return try ImageStore(path: root) + } catch { + fatalError("unable to initialize default ImageStore \(error)") + } + }() + + private static func defaultRoot() throws -> URL { + let root = FileManager.default.urls( + for: .applicationSupportDirectory, + in: .userDomainMask + ).first + guard let root else { + throw ContainerizationError(.notFound, message: "unable to get Application Support directory for current user") + } + return root.appendingPathComponent("com.apple.containerization") + + } } extension ImageStore { @@ -39,12 +70,20 @@ extension ImageStore { /// /// - Parameters: /// - reference: Name of the image. + /// - pull: Pull the image if it is not found. /// /// - Returns: A `Containerization.Image` object whose `reference` matches the given string. /// This method throws a `ContainerizationError(code: .notFound)` if the provided reference does not exist in the `ImageStore`. - public func get(reference: String) async throws -> Image { - let desc = try await self.referenceManager.get(reference: reference) - return Image(description: desc, contentStore: self.contentStore) + public func get(reference: String, pull: Bool = false) async throws -> Image { + do { + let desc = try await self.referenceManager.get(reference: reference) + return Image(description: desc, contentStore: self.contentStore) + } catch let error as ContainerizationError { + if error.code == .notFound && pull { + return try await self.pull(reference: reference) + } + throw error + } } /// Get a list of all images in the `ImageStore`. diff --git a/Sources/Containerization/VZVirtualMachineManager.swift b/Sources/Containerization/VZVirtualMachineManager.swift index f2fa38f1..29f4495e 100644 --- a/Sources/Containerization/VZVirtualMachineManager.swift +++ b/Sources/Containerization/VZVirtualMachineManager.swift @@ -30,7 +30,7 @@ public struct VZVirtualMachineManager: VirtualMachineManager { public init( kernel: Kernel, initialFilesystem: Mount, - bootlog: String?, + bootlog: String? = nil, logger: Logger? = nil ) { self.kernel = kernel diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index cb861dbc..94896b9c 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -95,7 +95,7 @@ struct IntegrationSuite: AsyncParsableCommand { .appendingPathComponent(name) } - func bootstrap() async throws -> (rootfs: Containerization.Mount, vmm: VirtualMachineManager) { + func bootstrap() async throws -> (rootfs: Containerization.Mount, vmm: VirtualMachineManager, image: Containerization.Image) { let reference = "ghcr.io/linuxcontainers/alpine:3.20" let store = Self.imageStore @@ -150,7 +150,8 @@ struct IntegrationSuite: AsyncParsableCommand { kernel: testKernel, initialFilesystem: initfs, bootlog: bootlog - ) + ), + image ) } @@ -209,6 +210,7 @@ struct IntegrationSuite: AsyncParsableCommand { "container hosts": testHostsFile, "container mount": testMounts, "nested virt": testNestedVirtualizationEnabled, + "container manager": testContainerManagerCreate, ] var passed = 0 diff --git a/Sources/Integration/VMTests.swift b/Sources/Integration/VMTests.swift index 6c5ff93d..7ad9bb53 100644 --- a/Sources/Integration/VMTests.swift +++ b/Sources/Integration/VMTests.swift @@ -80,6 +80,47 @@ extension IntegrationSuite { } } + func testContainerManagerCreate() async throws { + let id = "test-container-manager" + + // Get the kernel from bootstrap + let bs = try await bootstrap() + + // Create ContainerManager with kernel and initfs reference + let manager = try ContainerManager(vmm: bs.vmm) + defer { + try? manager.delete(id) + } + + let buffer = BufferWriter() + let container = try await manager.create( + id, + image: bs.image, + rootfs: bs.rootfs + ) { config in + config.process.arguments = ["/bin/echo", "ContainerManager test"] + config.process.stdout = buffer + } + + // Start the container + try await container.create() + try await container.start() + + // Wait for completion + let status = try await container.wait() + try await container.stop() + + guard status == 0 else { + throw IntegrationError.assert(msg: "process status \(status) != 0") + } + + let output = String(data: buffer.data, encoding: .utf8) + guard output == "ContainerManager test\n" else { + throw IntegrationError.assert( + msg: "process should have returned 'ContainerManager test' != '\(output ?? "nil")'") + } + } + private func createMountDirectory() throws -> URL { let dir = FileManager.default.uniqueTemporaryDirectory(create: true) try "hello".write(to: dir.appendingPathComponent("hi.txt"), atomically: true, encoding: .utf8) diff --git a/Sources/cctl/ContainerStore.swift b/Sources/cctl/ContainerStore.swift deleted file mode 100644 index 0c4f1ce2..00000000 --- a/Sources/cctl/ContainerStore.swift +++ /dev/null @@ -1,135 +0,0 @@ -//===----------------------------------------------------------------------===// -// Copyright © 2025 Apple Inc. and the Containerization project authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//===----------------------------------------------------------------------===// - -import Containerization -import ContainerizationError -import ContainerizationOCI -import ContainerizationOS -import Foundation - -#if os(macOS) - -import Crypto - -extension String { - fileprivate func hash() throws -> String { - guard let data = self.data(using: .utf8) else { - fatalError("\(self) could not be converted to Data") - } - return String(SHA256.hash(data: data).encoded.prefix(36)) - } -} - -#endif - -struct ContainerStore: Sendable { - private static let initImage = "vminit:latest" - - private let content: ContentStore - private let image: ImageStore - private let root: URL - - public let kernel: Kernel - public let initPath: URL - - public init(root: URL, kernel: Kernel, initPath: URL) async throws { - self.root = root - self.kernel = kernel - self.initPath = initPath - - let content = try LocalContentStore( - path: root.appendingPathComponent("content") - ) - self.content = content - self.image = try ImageStore( - path: root, - contentStore: content - ) - } - - public func fetch(reference: String) async throws -> Containerization.Image { - do { - return try await self.image.get(reference: reference) - } catch let error as ContainerizationError { - if error.code == .notFound { - return try await self.image.pull(reference: reference) - } - throw error - } - } - - public func create( - id: String, - reference: String, - fsSizeInBytes: UInt64, - configuration: (inout LinuxContainer.Configuration) throws -> Void - ) async throws -> LinuxContainer { - let initImage = try await image.getInitImage(reference: Self.initImage) - let initfs = try await { - do { - return try await initImage.initBlock(at: initPath, for: .linuxArm) - } catch let err as ContainerizationError { - guard err.code == .exists else { - throw err - } - return .block( - format: "ext4", - source: initPath.absolutePath(), - destination: "/", - options: ["ro"] - ) - } - }() - - let blockName = try reference.hash() + ".ext4" - let image = try await fetch(reference: reference) - let imageConfig = try await image.config(for: .current).config - - let imageBlock: Containerization.Mount = try await { - let source = self.root.appendingPathComponent(blockName) - do { - let unpacker = EXT4Unpacker(blockSizeInBytes: fsSizeInBytes) - return try await unpacker.unpack(image, for: .current, at: source) - } catch let err as ContainerizationError { - if err.code == .exists { - return .block( - format: "ext4", - source: source.absolutePath(), - destination: "/", - options: [] - ) - } - throw err - } catch { - throw error - } - }() - - let vmm = VZVirtualMachineManager( - kernel: kernel, - initialFilesystem: initfs, - bootlog: "cctl.log" - ) - - return try LinuxContainer(id, rootfs: imageBlock, vmm: vmm) { config in - if let imageConfig { - let process = LinuxContainer.Configuration.Process(from: imageConfig) - config.process = process - } - try configuration(&config) - } - } -} diff --git a/Sources/cctl/RunCommand.swift b/Sources/cctl/RunCommand.swift index 6d725bbd..d4d225ff 100644 --- a/Sources/cctl/RunCommand.swift +++ b/Sources/cctl/RunCommand.swift @@ -62,13 +62,6 @@ extension Application { }) public var kernel: String - @Option( - name: .customLong("init"), help: "Init block path", completion: .file(), - transform: { str in - URL(fileURLWithPath: str, relativeTo: .currentDirectory()).absoluteURL.path(percentEncoded: false) - }) - public var initBlock: String - @Option(name: .long, help: "Current working directory") var cwd: String = "/" @@ -79,10 +72,9 @@ extension Application { path: URL(fileURLWithPath: kernel), platform: .linuxArm ) - let store = try await ContainerStore( - root: Self.appRoot, + let manager = try await ContainerManager( kernel: kernel, - initPath: .init(filePath: initBlock) + initfsReference: "vminit:latest", ) let sigwinchStream = AsyncSignalHandler.create(notify: [SIGWINCH]) @@ -90,10 +82,10 @@ extension Application { try current.setraw() defer { current.tryReset() } - let container = try await store.create( - id: id, + let container = try await manager.create( + id, reference: imageReference, - fsSizeInBytes: fsSizeInMB.mib() + rootfsSizeInBytes: fsSizeInMB.mib() ) { config in config.cpus = cpus config.memoryInBytes = memory.mib() @@ -137,6 +129,10 @@ extension Application { config.hosts = hosts } + defer { + try? manager.delete(id) + } + try await container.create() try await container.start()