diff --git a/Package.swift b/Package.swift index 2c798095..8bbfe929 100644 --- a/Package.swift +++ b/Package.swift @@ -310,7 +310,6 @@ let package = Package( .product(name: "ContainerizationOS", package: "containerization"), .product(name: "ArgumentParser", package: "swift-argument-parser"), "ContainerAPIClient", - "ContainerNetworkServiceClient", "ContainerPersistence", "ContainerResource", "ContainerSandboxServiceClient", @@ -322,6 +321,7 @@ let package = Package( .target( name: "ContainerSandboxServiceClient", dependencies: [ + "ContainerAPIClient", "ContainerResource", "ContainerXPC", ], @@ -330,7 +330,10 @@ let package = Package( .target( name: "ContainerResource", dependencies: [ - .product(name: "Containerization", package: "containerization") + .product(name: "Containerization", package: "containerization"), + "ContainerXPC", + "CAuditToken", + "CVersion", ] ), .testTarget( diff --git a/Sources/ContainerCommands/Network/NetworkCreate.swift b/Sources/ContainerCommands/Network/NetworkCreate.swift index 716131ed..cf6f6d63 100644 --- a/Sources/ContainerCommands/Network/NetworkCreate.swift +++ b/Sources/ContainerCommands/Network/NetworkCreate.swift @@ -48,6 +48,12 @@ extension Application { }) var ipv6Subnet: CIDRv6? = nil + @Option(name: .long, help: "Set the plugin to use to create this network.") + var plugin: String = "container-network-vmnet" + + @Option(name: .long, help: "Set the variant of the network plugin to use.") + var pluginVariant: String? + @OptionGroup public var logOptions: Flags.Logging @@ -64,7 +70,8 @@ extension Application { mode: mode, ipv4Subnet: ipv4Subnet, ipv6Subnet: ipv6Subnet, - labels: parsedLabels + labels: parsedLabels, + pluginInfo: NetworkPluginInfo(plugin: self.plugin, variant: self.pluginVariant) ) let state = try await ClientNetwork.create(configuration: config) print(state.id) diff --git a/Sources/ContainerResource/Network/AllocatedAttachment.swift b/Sources/ContainerResource/Network/AllocatedAttachment.swift new file mode 100644 index 00000000..dfc68471 --- /dev/null +++ b/Sources/ContainerResource/Network/AllocatedAttachment.swift @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerXPC + +/// AllocatedAttachment represents a network attachment that has been allocated for use +/// by a container and any additional relevant data needed for a sandbox to properly +/// configure networking on container bootstrap. +public struct AllocatedAttachment: Sendable { + public let attachment: Attachment + public let additionalData: XPCMessage? + public let pluginInfo: NetworkPluginInfo + + public init(attachment: Attachment, additionalData: XPCMessage?, pluginInfo: NetworkPluginInfo) { + self.attachment = attachment + self.additionalData = additionalData + self.pluginInfo = pluginInfo + } +} diff --git a/Sources/ContainerResource/Network/Attachment.swift b/Sources/ContainerResource/Network/Attachment.swift index f7178f2d..96cee934 100644 --- a/Sources/ContainerResource/Network/Attachment.swift +++ b/Sources/ContainerResource/Network/Attachment.swift @@ -16,7 +16,7 @@ import ContainerizationExtras -/// A snapshot of a network interface allocated to a sandbox. +/// A snapshot of a network interface for a sandbox. public struct Attachment: Codable, Sendable { /// The network ID associated with the attachment. public let network: String diff --git a/Sources/ContainerResource/Network/NetworkConfiguration.swift b/Sources/ContainerResource/Network/NetworkConfiguration.swift index 234a0809..a5e277ca 100644 --- a/Sources/ContainerResource/Network/NetworkConfiguration.swift +++ b/Sources/ContainerResource/Network/NetworkConfiguration.swift @@ -18,6 +18,16 @@ import ContainerizationError import ContainerizationExtras import Foundation +public struct NetworkPluginInfo: Codable, Sendable, Hashable { + public let plugin: String + public let variant: String? + + public init(plugin: String, variant: String? = nil) { + self.plugin = plugin + self.variant = variant + } +} + /// Configuration parameters for network creation. public struct NetworkConfiguration: Codable, Sendable, Identifiable { /// A unique identifier for the network @@ -38,13 +48,19 @@ public struct NetworkConfiguration: Codable, Sendable, Identifiable { /// Key-value labels for the network. public var labels: [String: String] = [:] + /// Details about the network plugin that manages this network. + /// FIXME: This field only needs to be optional while we wait for the field + /// to be proliferated to most users when they update container. + public var pluginInfo: NetworkPluginInfo? + /// Creates a network configuration public init( id: String, mode: NetworkMode, ipv4Subnet: CIDRv4? = nil, ipv6Subnet: CIDRv6? = nil, - labels: [String: String] = [:] + labels: [String: String] = [:], + pluginInfo: NetworkPluginInfo, ) throws { self.id = id self.creationDate = Date() @@ -52,6 +68,7 @@ public struct NetworkConfiguration: Codable, Sendable, Identifiable { self.ipv4Subnet = ipv4Subnet self.ipv6Subnet = ipv6Subnet self.labels = labels + self.pluginInfo = pluginInfo try validate() } @@ -62,6 +79,7 @@ public struct NetworkConfiguration: Codable, Sendable, Identifiable { case ipv4Subnet case ipv6Subnet case labels + case pluginInfo // TODO: retain for deserialization compatability for now, remove later case subnet } @@ -81,6 +99,7 @@ public struct NetworkConfiguration: Codable, Sendable, Identifiable { ipv6Subnet = try container.decodeIfPresent(String.self, forKey: .ipv6Subnet) .map { try CIDRv6($0) } labels = try container.decodeIfPresent([String: String].self, forKey: .labels) ?? [:] + pluginInfo = try container.decodeIfPresent(NetworkPluginInfo.self, forKey: .pluginInfo) try validate() } @@ -94,6 +113,7 @@ public struct NetworkConfiguration: Codable, Sendable, Identifiable { try container.encodeIfPresent(ipv4Subnet, forKey: .ipv4Subnet) try container.encodeIfPresent(ipv6Subnet, forKey: .ipv6Subnet) try container.encode(labels, forKey: .labels) + try container.encodeIfPresent(pluginInfo, forKey: .pluginInfo) } private func validate() throws { diff --git a/Sources/ContainerResource/Network/NetworkState.swift b/Sources/ContainerResource/Network/NetworkState.swift index 3430d761..db5ba7d6 100644 --- a/Sources/ContainerResource/Network/NetworkState.swift +++ b/Sources/ContainerResource/Network/NetworkState.swift @@ -72,4 +72,10 @@ public enum NetworkState: Codable, Sendable { case .created(let config), .running(let config, _): config.labels.isBuiltin } } + + public var pluginInfo: NetworkPluginInfo? { + switch self { + case .created(let configuration), .running(let configuration, _): configuration.pluginInfo + } + } } diff --git a/Sources/ContainerXPC/XPCMessage.swift b/Sources/ContainerXPC/XPCMessage.swift index 18b3e98a..3c6a3dca 100644 --- a/Sources/ContainerXPC/XPCMessage.swift +++ b/Sources/ContainerXPC/XPCMessage.swift @@ -269,6 +269,12 @@ extension XPCMessage { } } + public func set(key: String, xpcDictionary: xpc_object_t) { + lock.withLock { + xpc_dictionary_set_value(self.object, key, xpcDictionary) + } + } + public func endpoint(key: String) -> xpc_endpoint_t? { lock.withLock { xpc_dictionary_get_value(self.object, key) diff --git a/Sources/Helpers/APIServer/APIServer+Start.swift b/Sources/Helpers/APIServer/APIServer+Start.swift index d9d38972..53175b91 100644 --- a/Sources/Helpers/APIServer/APIServer+Start.swift +++ b/Sources/Helpers/APIServer/APIServer+Start.swift @@ -67,6 +67,7 @@ extension APIServer { log: log, routes: &routes ) + await containersService.setNetworksService(networkService) initializeHealthCheckService(log: log, routes: &routes) try initializeKernelService(log: log, routes: &routes) let volumesService = try initializeVolumeService(containersService: containersService, log: log, routes: &routes) @@ -269,10 +270,12 @@ extension APIServer { .filter { $0.isBuiltin } .first if defaultNetwork == nil { + // FIXME: default network should be configurable elsewhere let config = try NetworkConfiguration( id: ClientNetwork.defaultNetworkName, mode: .nat, - labels: [ResourceLabelKeys.role: ResourceRoleValues.builtin] + labels: [ResourceLabelKeys.role: ResourceRoleValues.builtin], + pluginInfo: NetworkPluginInfo(plugin: "container-network-vmnet") ) _ = try await service.create(configuration: config) } diff --git a/Sources/Helpers/NetworkVmnet/NetworkVmnetHelper+Start.swift b/Sources/Helpers/NetworkVmnet/NetworkVmnetHelper+Start.swift index 94391a1d..e1aa3d62 100644 --- a/Sources/Helpers/NetworkVmnet/NetworkVmnetHelper+Start.swift +++ b/Sources/Helpers/NetworkVmnet/NetworkVmnetHelper+Start.swift @@ -76,11 +76,17 @@ extension NetworkVmnetHelper { log.info("configuring XPC server") let ipv4Subnet = try self.ipv4Subnet.map { try CIDRv4($0) } let ipv6Subnet = try self.ipv6Subnet.map { try CIDRv6($0) } + let pluginInfo = NetworkPluginInfo( + plugin: NetworkVmnetHelper._commandName, + variant: self.variant.rawValue + ) + let configuration = try NetworkConfiguration( id: id, mode: mode, ipv4Subnet: ipv4Subnet, ipv6Subnet: ipv6Subnet, + pluginInfo: pluginInfo ) let network = try Self.createNetwork( configuration: configuration, diff --git a/Sources/Helpers/RuntimeLinux/RuntimeLinuxHelper+Start.swift b/Sources/Helpers/RuntimeLinux/RuntimeLinuxHelper+Start.swift index 6c4a2676..3dc33152 100644 --- a/Sources/Helpers/RuntimeLinux/RuntimeLinuxHelper+Start.swift +++ b/Sources/Helpers/RuntimeLinux/RuntimeLinuxHelper+Start.swift @@ -60,19 +60,20 @@ extension RuntimeLinuxHelper { try adjustLimits() signal(SIGPIPE, SIG_IGN) - log.info("configuring XPC server") - let interfaceStrategy: any InterfaceStrategy + // FIXME: The network plugins that the runtime supports should be configurable elsewhere + var interfaceStrategies: [NetworkPluginInfo: InterfaceStrategy] = [ + NetworkPluginInfo(plugin: "container-network-vmnet", variant: "allocationOnly"): IsolatedInterfaceStrategy() + ] if #available(macOS 26, *) { - interfaceStrategy = NonisolatedInterfaceStrategy(log: log) - } else { - interfaceStrategy = IsolatedInterfaceStrategy() + interfaceStrategies[NetworkPluginInfo(plugin: "container-network-vmnet", variant: "reserved")] = NonisolatedInterfaceStrategy(log: log) } + log.info("configuring XPC server") nonisolated(unsafe) let anonymousConnection = xpc_connection_create(nil, nil) let server = SandboxService( root: .init(fileURLWithPath: root), - interfaceStrategy: interfaceStrategy, + interfaceStrategies: interfaceStrategies, eventLoopGroup: eventLoopGroup, connection: anonymousConnection, log: log diff --git a/Sources/Services/ContainerAPIService/Client/ClientNetwork.swift b/Sources/Services/ContainerAPIService/Client/ClientNetwork.swift index d7f239a2..3d37a69c 100644 --- a/Sources/Services/ContainerAPIService/Client/ClientNetwork.swift +++ b/Sources/Services/ContainerAPIService/Client/ClientNetwork.swift @@ -17,6 +17,7 @@ import ContainerResource import ContainerXPC import ContainerizationError +import ContainerizationExtras import ContainerizationOS import Foundation @@ -81,10 +82,10 @@ extension ClientNetwork { /// Delete the network with the given id. public static func delete(id: String) async throws { - let client = XPCClient(service: Self.serviceIdentifier) + let client = Self.newClient() let request = XPCMessage(route: .networkDelete) request.set(key: .networkId, value: id) - try await client.send(request) + let _ = try await xpcSend(client: client, message: request) } /// Retrieve the builtin network. diff --git a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift index f19288b2..af4accce 100644 --- a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift +++ b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift @@ -32,6 +32,7 @@ public actor ContainersService { struct ContainerState { var snapshot: ContainerSnapshot var client: SandboxClient? + var allocatedAttachments: [AllocatedAttachment] func getClient() throws -> SandboxClient { guard let client else { @@ -57,6 +58,9 @@ public actor ContainersService { private let lock = AsyncLock() private var containers: [String: ContainerState] + // FIXME: Find a better mechanism for services running on the APIServer to work with each other + private weak var networksService: NetworksService? + public init(appRoot: URL, pluginLoader: PluginLoader, log: Logger) throws { let containerRoot = appRoot.appendingPathComponent("containers") try FileManager.default.createDirectory(at: containerRoot, withIntermediateDirectories: true) @@ -68,6 +72,10 @@ public actor ContainersService { self.containers = try Self.loadAtBoot(root: containerRoot, loader: pluginLoader, log: log) } + public func setNetworksService(_ service: NetworksService) async { + self.networksService = service + } + static func loadAtBoot(root: URL, loader: PluginLoader, log: Logger) throws -> [String: ContainerState] { var directories = try FileManager.default.contentsOfDirectory( at: root, @@ -89,7 +97,8 @@ public actor ContainersService { status: .stopped, networks: [], startedDate: nil - ) + ), + allocatedAttachments: [] ) results[config.id] = state guard runtimePlugins.first(where: { $0.name == config.runtimeHandler }) != nil else { @@ -282,7 +291,7 @@ public actor ContainersService { networks: [], startedDate: nil ) - await self.setContainerState(configuration.id, ContainerState(snapshot: snapshot), context: context) + await self.setContainerState(configuration.id, ContainerState(snapshot: snapshot, allocatedAttachments: []), context: context) } catch { throw error } @@ -305,7 +314,20 @@ public actor ContainersService { let path = self.containerRoot.appendingPathComponent(id) let config = try Self.getContainerConfiguration(at: path) + var allocatedAttachments = [AllocatedAttachment]() do { + for n in config.networks { + let allocatedAttach = try await self.networksService?.allocate( + id: n.network, + hostname: n.options.hostname, + macAddress: n.options.macAddress + ) + guard let allocatedAttach = allocatedAttach else { + throw ContainerizationError(.internalError, message: "failed to allocate a network") + } + allocatedAttachments.append(allocatedAttach) + } + try Self.registerService( plugin: self.runtimePlugins.first { $0.name == config.runtimeHandler }!, loader: self.pluginLoader, @@ -318,8 +340,7 @@ public actor ContainersService { id: id, runtime: runtime ) - - try await sandboxClient.bootstrap(stdio: stdio) + try await sandboxClient.bootstrap(stdio: stdio, allocatedAttachments: allocatedAttachments) try await self.exitMonitor.registerProcess( id: id, @@ -327,8 +348,17 @@ public actor ContainersService { ) state.client = sandboxClient + state.allocatedAttachments = allocatedAttachments await self.setContainerState(id, state, context: context) } catch { + for allocatedAttach in allocatedAttachments { + do { + try await self.networksService?.deallocate(attachment: allocatedAttach.attachment) + } catch { + self.log.error("failed to deallocate network attachment in \(id) for \(allocatedAttach.attachment.network): \(error)") + } + } + let label = Self.fullLaunchdServiceLabel( runtimeName: config.runtimeHandler, instanceId: id @@ -595,9 +625,21 @@ public actor ContainersService { self.log.error("Failed to deregister sandbox service for \(id): \(error)") } + // Best effort deallocate network attachments for the container. Don't throw on + // failure so we can continue with state cleanup. + self.log.info("Deallocating network attachments for \(id)") + for allocatedAttach in state.allocatedAttachments { + do { + try await self.networksService?.deallocate(attachment: allocatedAttach.attachment) + } catch { + self.log.error("failed to deallocate network attachment in \(id) for \(allocatedAttach.attachment.network): \(error)") + } + } + state.snapshot.status = .stopped state.snapshot.networks = [] state.client = nil + state.allocatedAttachments = [] await self.setContainerState(id, state, context: context) let options = try getContainerCreationOptions(id: id) diff --git a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift index 5eef5cd0..90ac19fc 100644 --- a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift +++ b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift @@ -19,6 +19,7 @@ import ContainerNetworkServiceClient import ContainerPersistence import ContainerPlugin import ContainerResource +import ContainerXPC import Containerization import ContainerizationError import ContainerizationExtras @@ -27,16 +28,23 @@ import Foundation import Logging public actor NetworksService { + struct NetworkServiceState { + var networkState: NetworkState + var client: NetworkClient + } + private let pluginLoader: PluginLoader private let resourceRoot: URL private let containersService: ContainersService private let log: Logger private let store: FilesystemEntityStore - private let networkPlugin: Plugin - private var networkStates = [String: NetworkState]() + private let networkPlugins: [Plugin] private var busyNetworks = Set() + private let stateLock = AsyncLock() + private var serviceStates = [String: NetworkServiceState]() + public init( pluginLoader: PluginLoader, resourceRoot: URL, @@ -55,15 +63,14 @@ public actor NetworksService { log: log ) - let networkPlugin = + let networkPlugins = pluginLoader .findPlugins() .filter { $0.hasType(.network) } - .first - guard let networkPlugin else { - throw ContainerizationError(.internalError, message: "cannot find network plugin") + guard !networkPlugins.isEmpty else { + throw ContainerizationError(.internalError, message: "cannot find any plugins with type network") } - self.networkPlugin = networkPlugin + self.networkPlugins = networkPlugins let configurations = try await store.list() for var configuration in configurations { @@ -76,34 +83,55 @@ public actor NetworksService { } } + // Ensure that the network always has plugin information. + // Before this field was added, the code always assumed we were using the + // container-network-vmnet network plugin, so it should be safe to fallback to that + // if no info was found in an on disk configuration. + if configuration.pluginInfo == nil { + configuration.pluginInfo = NetworkPluginInfo(plugin: "container-network-vmnet") + try await store.update(configuration) + } + // Start up the network. do { try await registerService(configuration: configuration) } catch { log.error( - "failed to start network", + "failed to start network: \(error)", metadata: [ "id": "\(configuration.id)" ]) } - let client = NetworkClient(id: configuration.id) - let networkState = try await client.state() + let client = try Self.getClient(configuration: configuration) + var networkState = try await client.state() // FIXME: Temporary workaround for persisted configuration being overwritten // by what comes back from the network helper, which messes up creationDate. + // FIXME: Temporarily need to override the plugin information with the info from + // the helper, so we can ensure that older networks get a variant value. + var finalConfig = configuration switch networkState { - case .created(_): - networkStates[configuration.id] = NetworkState.created(configuration) - case .running(_, let status): - networkStates[configuration.id] = NetworkState.running(configuration, status) + case .created(let helperConfig): + finalConfig.pluginInfo = helperConfig.pluginInfo + networkState = NetworkState.created(finalConfig) + case .running(let helperConfig, let status): + finalConfig.pluginInfo = helperConfig.pluginInfo + networkState = NetworkState.running(finalConfig, status) } + let state = NetworkServiceState( + networkState: networkState, + client: client + ) + + serviceStates[finalConfig.id] = state + guard case .running = networkState else { log.error( "network failed to start", metadata: [ - "id": "\(configuration.id)", + "id": "\(finalConfig.id)", "state": "\(networkState.state)", ]) return @@ -114,8 +142,8 @@ public actor NetworksService { /// List all networks registered with the service. public func list() async throws -> [NetworkState] { log.info("network service: list") - return networkStates.reduce(into: [NetworkState]()) { - $0.append($1.value) + return serviceStates.reduce(into: [NetworkState]()) { + $0.append($1.value.networkState) } } @@ -141,41 +169,45 @@ public actor NetworksService { defer { busyNetworks.remove(configuration.id) } // Ensure the network doesn't already exist. - guard networkStates[configuration.id] == nil else { - throw ContainerizationError(.exists, message: "network \(configuration.id) already exists") - } - - // Create and start the network. - try await registerService(configuration: configuration) - let client = NetworkClient(id: configuration.id) - - // Ensure the network is running, and set up the persistent network state - // using our configuration data, as the one from the helper doesn't include - // metadata. - guard case .running(_, let status) = try await client.state() else { - throw ContainerizationError(.invalidState, message: "network \(configuration.id) failed to start") - } - let networkState: NetworkState = .running(configuration, status) - networkStates[configuration.id] = networkState - - // Persist the configuration data. - do { - try await store.create(configuration) - return networkState - } catch { - networkStates.removeValue(forKey: configuration.id) - do { - try pluginLoader.deregisterWithLaunchd(plugin: networkPlugin, instanceId: configuration.id) - } catch { - log.error( - "failed to deregister network service after failed creation", - metadata: [ - "id": "\(configuration.id)", - "error": "\(error.localizedDescription)", - ]) + return try await self.stateLock.withLock { _ in + guard await self.serviceStates[configuration.id] == nil else { + throw ContainerizationError(.exists, message: "network \(configuration.id) already exists") } - throw error + // Create and start the network. + try await self.registerService(configuration: configuration) + let client = try Self.getClient(configuration: configuration) + + // Ensure the network is running, and set up the persistent network state + // using our configuration data + guard case .running(let helperConfig, let status) = try await client.state() else { + throw ContainerizationError(.invalidState, message: "network \(configuration.id) failed to start") + } + var finalConfig = configuration + finalConfig.pluginInfo = helperConfig.pluginInfo + + let networkState: NetworkState = .running(finalConfig, status) + let serviceState = NetworkServiceState(networkState: networkState, client: client) + await self.setServiceState(key: finalConfig.id, value: serviceState) + + // Persist the configuration data. + do { + try await self.store.create(finalConfig) + return networkState + } catch { + await self.removeServiceState(key: finalConfig.id) + do { + try await self.deregisterService(configuration: finalConfig) + } catch { + self.log.error( + "failed to deregister network service after failed creation", + metadata: [ + "id": "\(finalConfig.id)", + "error": "\(error.localizedDescription)", + ]) + } + throw error + } } } @@ -196,86 +228,117 @@ public actor NetworksService { "id": "\(id)" ]) - guard let networkState = networkStates[id] else { - throw ContainerizationError(.notFound, message: "no network for id \(id)") - } + try await stateLock.withLock { _ in + guard let serviceState = await self.serviceStates[id] else { + throw ContainerizationError(.notFound, message: "no network for id \(id)") + } - // basic sanity checks on network itself - if networkState.isBuiltin { - throw ContainerizationError(.invalidArgument, message: "cannot delete builtin network: \(id)") - } + guard case .running(let netConfig, _) = serviceState.networkState else { + throw ContainerizationError(.invalidState, message: "cannot delete network \(id) in state \(serviceState.networkState.state)") + } - guard case .running = networkState else { - throw ContainerizationError(.invalidState, message: "cannot delete network \(id) in state \(networkState.state)") - } + // basic sanity checks on network itself + if serviceState.networkState.isBuiltin { + throw ContainerizationError(.invalidArgument, message: "cannot delete builtin network: \(id)") + } - // prevent container operations while we atomically check and delete - try await containersService.withContainerList { containers in - // find all containers that refer to the network - var referringContainers = Set() - for container in containers { - for attachmentConfiguration in container.configuration.networks { - if attachmentConfiguration.network == id { - referringContainers.insert(container.configuration.id) - break + // prevent container operations while we atomically check and delete + try await self.containersService.withContainerList { containers in + // find all containers that refer to the network + var referringContainers = Set() + for container in containers { + for attachmentConfiguration in container.configuration.networks { + if attachmentConfiguration.network == id { + referringContainers.insert(container.configuration.id) + break + } } } + + // bail if any referring containers + guard referringContainers.isEmpty else { + throw ContainerizationError( + .invalidState, + message: "cannot delete subnet \(id) with referring containers: \(referringContainers.joined(separator: ", "))" + ) + } + + // disable the allocator so nothing else can attach + // TODO: remove this from the network helper later, not necesssary now that withContainerList is here + guard try await serviceState.client.disableAllocator() else { + throw ContainerizationError(.invalidState, message: "cannot delete subnet \(id) because the IP allocator cannot be disabled with active containers") + } + + // start network deletion, this is the last place we'll want to throw + do { + try await self.deregisterService(configuration: netConfig) + } catch { + self.log.error( + "failed to deregister network service", + metadata: [ + "id": "\(id)", + "error": "\(error.localizedDescription)", + ]) + } + + // deletion is underway, do not throw anything now + do { + try await self.store.delete(id) + } catch { + self.log.error( + "failed to delete network from configuration store", + metadata: [ + "id": "\(id)", + "error": "\(error.localizedDescription)", + ]) + } } - // bail if any referring containers - guard referringContainers.isEmpty else { - throw ContainerizationError( - .invalidState, - message: "cannot delete subnet \(id) with referring containers: \(referringContainers.joined(separator: ", "))" - ) - } - - // disable the allocator so nothing else can attach - // TODO: remove this from the network helper later, not necesssary now that withContainerList is here - let client = NetworkClient(id: id) - guard try await client.disableAllocator() else { - throw ContainerizationError(.invalidState, message: "cannot delete subnet \(id) because the IP allocator cannot be disabled with active containers") - } - - // start network deletion, this is the last place we'll want to throw - do { - try self.pluginLoader.deregisterWithLaunchd(plugin: self.networkPlugin, instanceId: id) - } catch { - self.log.error( - "failed to deregister network service", - metadata: [ - "id": "\(id)", - "error": "\(error.localizedDescription)", - ]) - } - - // deletion is underway, do not throw anything now - do { - try await self.store.delete(id) - } catch { - self.log.error( - "failed to delete network from configuration store", - metadata: [ - "id": "\(id)", - "error": "\(error.localizedDescription)", - ]) - } + // having deleted successfully, remove the runtime state + await self.removeServiceState(key: id) } - - // having deleted successfully, remove the runtime state - self.networkStates.removeValue(forKey: id) } /// Perform a hostname lookup on all networks. public func lookup(hostname: String) async throws -> Attachment? { - for id in networkStates.keys { - let client = NetworkClient(id: id) - guard let allocation = try await client.lookup(hostname: hostname) else { - continue + try await self.stateLock.withLock { _ in + for state in await self.serviceStates.values { + guard let allocation = try await state.client.lookup(hostname: hostname) else { + continue + } + return allocation } - return allocation + return nil } - return nil + } + + public func allocate(id: String, hostname: String, macAddress: MACAddress?) async throws -> AllocatedAttachment { + guard let serviceState = serviceStates[id] else { + throw ContainerizationError(.notFound, message: "no network for id \(id)") + } + guard let pluginInfo = serviceState.networkState.pluginInfo else { + throw ContainerizationError(.internalError, message: "network \(id) missing plugin information") + } + let (attach, additionalData) = try await serviceState.client.allocate(hostname: hostname, macAddress: macAddress) + return AllocatedAttachment( + attachment: attach, + additionalData: additionalData, + pluginInfo: pluginInfo + ) + } + + public func deallocate(attachment: Attachment) async throws { + guard let serviceState = serviceStates[attachment.network] else { + throw ContainerizationError(.notFound, message: "no network for id \(attachment.network)") + } + return try await serviceState.client.deallocate(hostname: attachment.hostname) + } + + private static func getClient(configuration: NetworkConfiguration) throws -> NetworkClient { + guard let pluginInfo = configuration.pluginInfo else { + throw ContainerizationError(.internalError, message: "network \(configuration.id) missing plugin information") + } + return NetworkClient(id: configuration.id, plugin: pluginInfo.plugin) } private func registerService(configuration: NetworkConfiguration) async throws { @@ -283,6 +346,17 @@ public actor NetworksService { throw ContainerizationError(.invalidArgument, message: "unsupported network mode \(configuration.mode.rawValue)") } + guard let pluginInfo = configuration.pluginInfo else { + throw ContainerizationError(.internalError, message: "network \(configuration.id) missing plugin information") + } + + guard let networkPlugin = self.networkPlugins.first(where: { $0.name == pluginInfo.plugin }) else { + throw ContainerizationError( + .notFound, + message: "unable to locate network plugin \(pluginInfo.plugin)" + ) + } + guard let serviceIdentifier = networkPlugin.getMachService(instanceId: configuration.id, type: .network) else { throw ContainerizationError(.invalidArgument, message: "unsupported network mode \(configuration.mode.rawValue)") } @@ -298,8 +372,8 @@ public actor NetworksService { if let ipv4Subnet = configuration.ipv4Subnet { var existingCidrs: [CIDRv4] = [] - for networkState in networkStates.values { - if case .running(_, let status) = networkState { + for serviceState in serviceStates.values { + if case .running(_, let status) = serviceState.networkState { existingCidrs.append(status.ipv4Subnet) } } @@ -318,8 +392,8 @@ public actor NetworksService { if let ipv6Subnet = configuration.ipv6Subnet { var existingCidrs: [CIDRv6] = [] - for networkState in networkStates.values { - if case .running(_, let status) = networkState, let otherIPv6Subnet = status.ipv6Subnet { + for serviceState in serviceStates.values { + if case .running(_, let status) = serviceState.networkState, let otherIPv6Subnet = status.ipv6Subnet { existingCidrs.append(otherIPv6Subnet) } } @@ -336,6 +410,10 @@ public actor NetworksService { args += ["--subnet-v6", ipv6Subnet.description] } + if let variant = configuration.pluginInfo?.variant { + args += ["--variant", variant] + } + try await pluginLoader.registerWithLaunchd( plugin: networkPlugin, pluginStateRoot: store.entityUrl(configuration.id), @@ -343,4 +421,27 @@ public actor NetworksService { instanceId: configuration.id ) } + + private func deregisterService(configuration: NetworkConfiguration) async throws { + guard let pluginInfo = configuration.pluginInfo else { + throw ContainerizationError(.internalError, message: "network \(configuration.id) missing plugin information") + } + guard let networkPlugin = self.networkPlugins.first(where: { $0.name == pluginInfo.plugin }) else { + throw ContainerizationError( + .notFound, + message: "unable to locate network plugin \(pluginInfo.plugin)" + ) + } + try self.pluginLoader.deregisterWithLaunchd(plugin: networkPlugin, instanceId: configuration.id) + } +} + +extension NetworksService { + private func removeServiceState(key: String) { + self.serviceStates.removeValue(forKey: key) + } + + private func setServiceState(key: String, value: NetworkServiceState) { + self.serviceStates[key] = value + } } diff --git a/Sources/Services/ContainerNetworkService/Client/NetworkClient.swift b/Sources/Services/ContainerNetworkService/Client/NetworkClient.swift index 69b8207e..0d5aac9e 100644 --- a/Sources/Services/ContainerNetworkService/Client/NetworkClient.swift +++ b/Sources/Services/ContainerNetworkService/Client/NetworkClient.swift @@ -22,18 +22,23 @@ import Foundation /// A client for interacting with a single network. public struct NetworkClient: Sendable { - // FIXME: need more flexibility than a hard-coded constant? - static let label = "com.apple.container.network.container-network-vmnet" + static let label = "com.apple.container.network" + + public static func machServiceLabel(id: String, plugin: String) -> String { + "\(Self.label).\(plugin).\(id)" + } private var machServiceLabel: String { - "\(Self.label).\(id)" + Self.machServiceLabel(id: id, plugin: plugin) } let id: String + let plugin: String /// Create a client for a network. - public init(id: String) { + public init(id: String, plugin: String) { self.id = id + self.plugin = plugin } } diff --git a/Sources/Services/ContainerSandboxService/Client/SandboxClient.swift b/Sources/Services/ContainerSandboxService/Client/SandboxClient.swift index ca61eee4..76df65a1 100644 --- a/Sources/Services/ContainerSandboxService/Client/SandboxClient.swift +++ b/Sources/Services/ContainerSandboxService/Client/SandboxClient.swift @@ -14,6 +14,7 @@ // limitations under the License. //===----------------------------------------------------------------------===// +import ContainerAPIClient import ContainerResource import ContainerXPC import Containerization @@ -76,7 +77,7 @@ public struct SandboxClient: Sendable { // Runtime Methods extension SandboxClient { - public func bootstrap(stdio: [FileHandle?]) async throws { + public func bootstrap(stdio: [FileHandle?], allocatedAttachments: [AllocatedAttachment]) async throws { let request = XPCMessage(route: SandboxRoutes.bootstrap.rawValue) for (i, h) in stdio.enumerated() { @@ -96,6 +97,7 @@ extension SandboxClient { } do { + try request.setAllocatedAttachments(allocatedAttachments) try await self.client.send(request) } catch { throw ContainerizationError( @@ -322,4 +324,26 @@ extension XPCMessage { } return try JSONDecoder().decode(SandboxSnapshot.self, from: data) } + + func setAllocatedAttachments(_ allocatedAttachments: [AllocatedAttachment]) throws { + let encoder = JSONEncoder() + let allocatedAttachmentsArray = xpc_array_create_empty() + for allocatedAttach in allocatedAttachments { + let xpcObject: xpc_object_t = xpc_dictionary_create_empty() + let networkXPC = XPCMessage(object: xpcObject) + + let attachmentEncoded = try encoder.encode(allocatedAttach.attachment) + networkXPC.set(key: SandboxKeys.networkAttachment.rawValue, value: attachmentEncoded) + + let pluginInfoEncoded = try encoder.encode(allocatedAttach.pluginInfo) + networkXPC.set(key: SandboxKeys.networkPluginInfo.rawValue, value: pluginInfoEncoded) + + if let additionalData = allocatedAttach.additionalData { + xpc_dictionary_set_value(networkXPC.underlying, SandboxKeys.networkAdditionalData.rawValue, additionalData.underlying) + } + + xpc_array_append_value(allocatedAttachmentsArray, networkXPC.underlying) + } + self.set(key: SandboxKeys.allocatedAttachments.rawValue, value: allocatedAttachmentsArray) + } } diff --git a/Sources/Services/ContainerSandboxService/Client/SandboxKeys.swift b/Sources/Services/ContainerSandboxService/Client/SandboxKeys.swift index 2cb5b5ff..e207cb04 100644 --- a/Sources/Services/ContainerSandboxService/Client/SandboxKeys.swift +++ b/Sources/Services/ContainerSandboxService/Client/SandboxKeys.swift @@ -42,4 +42,10 @@ public enum SandboxKeys: String { /// Container statistics case statistics + + /// Network resource keys. + case allocatedAttachments + case networkAdditionalData + case networkAttachment + case networkPluginInfo } diff --git a/Sources/Services/ContainerSandboxService/Server/SandboxService.swift b/Sources/Services/ContainerSandboxService/Server/SandboxService.swift index 70ef364a..da9d0bf6 100644 --- a/Sources/Services/ContainerSandboxService/Server/SandboxService.swift +++ b/Sources/Services/ContainerSandboxService/Server/SandboxService.swift @@ -14,7 +14,7 @@ // limitations under the License. //===----------------------------------------------------------------------===// -import ContainerNetworkServiceClient +import ContainerAPIClient import ContainerPersistence import ContainerResource import ContainerSandboxServiceClient @@ -39,7 +39,7 @@ import struct ContainerizationOCI.Process public actor SandboxService { private let connection: xpc_connection_t private let root: URL - private let interfaceStrategy: InterfaceStrategy + private let interfaceStrategies: [NetworkPluginInfo: InterfaceStrategy] private var container: ContainerInfo? private let monitor: ExitMonitor private let eventLoopGroup: any EventLoopGroup @@ -62,13 +62,13 @@ public actor SandboxService { public init( root: URL, - interfaceStrategy: InterfaceStrategy, + interfaceStrategies: [NetworkPluginInfo: InterfaceStrategy], eventLoopGroup: any EventLoopGroup, connection: xpc_connection_t, log: Logger ) { self.root = root - self.interfaceStrategy = interfaceStrategy + self.interfaceStrategies = interfaceStrategies self.log = log self.monitor = ExitMonitor(log: log) self.eventLoopGroup = eventLoopGroup @@ -130,9 +130,11 @@ public actor SandboxService { logger: self.log ) + let allocatedAttachments = try message.getAllocatedAttachments() + // Dynamically configure the DNS nameserver from a network if no explicit configuration if let dns = config.dns, dns.nameservers.isEmpty { - let defaultNameservers = try await self.getDefaultNameservers(attachmentConfigurations: config.networks) + let defaultNameservers = try await self.getDefaultNameservers(allocatedAttachments: allocatedAttachments) if !defaultNameservers.isEmpty { config.dns = ContainerConfiguration.DNSConfiguration( nameservers: defaultNameservers, @@ -145,16 +147,19 @@ public actor SandboxService { var attachments: [Attachment] = [] var interfaces: [Interface] = [] - for index in 0.. [String] { - for attachmentConfiguration in attachmentConfigurations { - let client = NetworkClient(id: attachmentConfiguration.network) - let state = try await client.state() + private func getDefaultNameservers(allocatedAttachments: [AllocatedAttachment]) async throws -> [String] { + for allocatedAttach in allocatedAttachments { + let state = try await ClientNetwork.get(id: allocatedAttach.attachment.network) guard case .running(_, let status) = state else { continue } @@ -1035,16 +1039,7 @@ public actor SandboxService { self.log.error("failed to stop container during cleanup: \(error)") } - // Give back our lovely IP(s) await self.stopSocketForwarders() - for attachment in containerInfo.attachments { - let client = NetworkClient(id: attachment.network) - do { - try await client.deallocate(hostname: attachment.hostname) - } catch { - self.log.error("failed to deallocate hostname \(attachment.hostname) on network \(attachment.network) during cleanup: \(error)") - } - } let status = exitStatus ?? ExitStatus(exitCode: 255) let waiters = self.waiters[id] ?? [] @@ -1096,6 +1091,50 @@ extension XPCMessage { } return try JSONDecoder().decode(ProcessConfiguration.self, from: data) } + + fileprivate func getAllocatedAttachments() throws -> [AllocatedAttachment] { + guard let attachmentArray = xpc_dictionary_get_value(self.underlying, SandboxKeys.allocatedAttachments.rawValue) else { + throw ContainerizationError(.invalidArgument, message: "missing allocatedAttachments array in message") + } + + var results = [AllocatedAttachment]() + let decoder = JSONDecoder() + + let arrayCount = xpc_array_get_count(attachmentArray) + + for i in 0..