From b4669596446f8ea4397a7e703d217bb1d7e94204 Mon Sep 17 00:00:00 2001 From: J Logan Date: Tue, 12 May 2026 16:34:06 -0700 Subject: [PATCH] Remove IP allocation from API server. (#1545) - Closes #1318. - Closes #1378. - Reduces the complexity and coupling for IP allocation. - Runtimes connect to networks for the life of the running container. The runtime shuts down on connection loss. - Networks automatically deallocate a runtime's IP address and hostname record on connection loss. - Removes AllocatedAttachment as this is no longer necessary. The `bootstrap()` XPC now takes a `NetworkBootstrapInfo` array which parallels the attachments in the bundle config and provides the network plugin attributes needed to create VM network interface configurations. ## Type of Change - [x] Bug fix - [ ] New feature - [ ] Breaking change - [ ] Documentation update ## Motivation and Context Simplify IP allocation and make deallocation more reliable. ## Testing - [x] Tested locally - [ ] Added/updated tests - [ ] Added/updated docs --- Package.swift | 2 +- .../Server/Containers/ContainersService.swift | 64 ++------- .../Server/Networks/NetworksService.swift | 17 +-- .../Server/NetworkService.swift | 7 +- .../Client/NetworkBootstrapInfo.swift} | 17 +-- .../Client/SandboxClient.swift | 28 +--- .../Client/SandboxKeys.swift | 7 +- .../Server/SandboxService.swift | 131 ++++++++---------- 8 files changed, 90 insertions(+), 183 deletions(-) rename Sources/{ContainerResource/Network/AllocatedAttachment.swift => Services/ContainerSandboxService/Client/NetworkBootstrapInfo.swift} (61%) diff --git a/Package.swift b/Package.swift index 9df3828c..cf3f1aa5 100644 --- a/Package.swift +++ b/Package.swift @@ -358,7 +358,7 @@ let package = Package( .product(name: "ContainerizationExtras", package: "containerization"), .product(name: "ContainerizationOS", package: "containerization"), .product(name: "ArgumentParser", package: "swift-argument-parser"), - "ContainerAPIClient", + "ContainerNetworkServiceClient", "ContainerOS", "ContainerPersistence", "ContainerResource", diff --git a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift index 940bf879..77c4e762 100644 --- a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift +++ b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift @@ -34,9 +34,7 @@ import SystemPackage public actor ContainersService { struct ContainerState { var snapshot: ContainerSnapshot - var client: SandboxClient? - var allocatedAttachments: [AllocatedAttachment] - var networkSessions: [XPCClientSession] + var client: SandboxClient? = nil func getClient() throws -> SandboxClient { guard let client else { @@ -133,8 +131,6 @@ public actor ContainersService { networks: [], startedDate: nil ), - allocatedAttachments: [], - networkSessions: [] ) results[config.id] = state guard runtimePlugins.first(where: { $0.name == config.runtimeHandler }) != nil else { @@ -396,7 +392,7 @@ public actor ContainersService { networks: [], startedDate: nil ) - await self.setContainerState(configuration.id, ContainerState(snapshot: snapshot, allocatedAttachments: [], networkSessions: []), context: context) + await self.setContainerState(configuration.id, ContainerState(snapshot: snapshot), context: context) } catch { throw error } @@ -436,41 +432,15 @@ public actor ContainersService { let path = self.containerRoot.appendingPathComponent(id) let (config, _) = try Self.getContainerConfiguration(at: path) - var allocatedAttachments = [AllocatedAttachment]() - var networkSessions = [XPCClientSession]() - do { - for n in config.networks { - guard - let (allocatedAttach, session) = try await self.networksService?.allocate( - id: n.network, - hostname: n.options.hostname, - macAddress: n.options.macAddress - ) - else { - throw ContainerizationError(.internalError, message: "failed to allocate a network") - } - - var finalAttach = allocatedAttach - if let mtu = n.options.mtu { - let a = allocatedAttach.attachment - finalAttach = AllocatedAttachment( - attachment: Attachment( - network: a.network, - hostname: a.hostname, - ipv4Address: a.ipv4Address, - ipv4Gateway: a.ipv4Gateway, - ipv6Address: a.ipv6Address, - macAddress: a.macAddress, - mtu: mtu - ), - additionalData: allocatedAttach.additionalData, - pluginInfo: allocatedAttach.pluginInfo - ) - } - allocatedAttachments.append(finalAttach) - networkSessions.append(session) + var networkBootstrapInfos = [NetworkBootstrapInfo]() + for n in config.networks { + guard let pluginInfo = try await self.networksService?.pluginInfo(id: n.network) else { + throw ContainerizationError(.internalError, message: "failed to get plugin info for network \(n.network)") } + networkBootstrapInfos.append(NetworkBootstrapInfo(pluginInfo: pluginInfo)) + } + do { try Self.registerService( plugin: self.runtimePlugins.first { $0.name == config.runtimeHandler }!, loader: self.pluginLoader, @@ -484,7 +454,7 @@ public actor ContainersService { id: id, runtime: runtime ) - try await sandboxClient.bootstrap(stdio: stdio, allocatedAttachments: allocatedAttachments, dynamicEnv: dynamicEnv) + try await sandboxClient.bootstrap(stdio: stdio, networkBootstrapInfos: networkBootstrapInfos, dynamicEnv: dynamicEnv) try await self.exitMonitor.registerProcess( id: id, @@ -492,14 +462,8 @@ public actor ContainersService { ) state.client = sandboxClient - state.allocatedAttachments = allocatedAttachments - state.networkSessions = networkSessions await self.setContainerState(id, state, context: context) } catch { - for session in networkSessions { - session.close() - } - let label = Self.fullLaunchdServiceLabel( runtimeName: config.runtimeHandler, instanceId: id @@ -994,17 +958,9 @@ public actor ContainersService { ]) } - // Close network sessions — the network helper auto-releases allocations on disconnect. - self.log.info("closing network sessions", metadata: ["id": "\(id)"]) - for session in state.networkSessions { - session.close() - } - state.snapshot.status = .stopped state.snapshot.networks = [] state.client = nil - state.allocatedAttachments = [] - state.networkSessions = [] await self.setContainerState(id, state, context: context) let options = try getContainerCreationOptions(id: id) diff --git a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift index 1ce3ee90..271f0719 100644 --- a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift +++ b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift @@ -19,7 +19,6 @@ import ContainerNetworkServiceClient import ContainerPersistence import ContainerPlugin import ContainerResource -import ContainerXPC import Containerization import ContainerizationError import ContainerizationExtras @@ -380,26 +379,14 @@ public actor NetworksService { } } - public func allocate(id: String, hostname: String, macAddress: MACAddress?) async throws -> (AllocatedAttachment, XPCClientSession) { + public func pluginInfo(id: String) throws -> NetworkPluginInfo { guard let serviceState = serviceStates[id] else { throw ContainerizationError(.notFound, message: "no network for id \(id)") } guard let pluginInfo = serviceState.networkState.pluginInfo else { throw ContainerizationError(.internalError, message: "network \(id) missing plugin information") } - let session = serviceState.client.connect() - do { - let (attach, additionalData) = try await serviceState.client.allocate(hostname: hostname, macAddress: macAddress, on: session) - let alloc = AllocatedAttachment( - attachment: attach, - additionalData: additionalData, - pluginInfo: pluginInfo - ) - return (alloc, session) - } catch { - session.close() - throw error - } + return pluginInfo } private static func getClient(configuration: NetworkConfiguration) throws -> ContainerNetworkServiceClient.NetworkClient { diff --git a/Sources/Services/ContainerNetworkService/Server/NetworkService.swift b/Sources/Services/ContainerNetworkService/Server/NetworkService.swift index 1f7ee8bd..8d12ae76 100644 --- a/Sources/Services/ContainerNetworkService/Server/NetworkService.swift +++ b/Sources/Services/ContainerNetworkService/Server/NetworkService.swift @@ -27,7 +27,7 @@ public actor NetworkService: Sendable { private let log: Logger private var allocator: AttachmentAllocator private var macAddresses: [UInt32: MACAddress] - private var allocationsBySession: [XPCServerSession: [(hostname: String, index: UInt32)]] = [:] + private var allocationsBySession: [XPCServerSession: [(hostname: String, index: UInt32)]] /// Set up a network service for the specified network. public init( @@ -42,10 +42,11 @@ public actor NetworkService: Sendable { let subnet = status.ipv4Subnet let size = Int(subnet.upper.value - subnet.lower.value - 3) - self.allocator = try AttachmentAllocator(lower: subnet.lower.value + 2, size: size) - self.macAddresses = [:] self.network = network self.log = log + self.allocator = try AttachmentAllocator(lower: subnet.lower.value + 2, size: size) + self.macAddresses = [:] + self.allocationsBySession = [:] } @Sendable diff --git a/Sources/ContainerResource/Network/AllocatedAttachment.swift b/Sources/Services/ContainerSandboxService/Client/NetworkBootstrapInfo.swift similarity index 61% rename from Sources/ContainerResource/Network/AllocatedAttachment.swift rename to Sources/Services/ContainerSandboxService/Client/NetworkBootstrapInfo.swift index dfc68471..613e9b6a 100644 --- a/Sources/ContainerResource/Network/AllocatedAttachment.swift +++ b/Sources/Services/ContainerSandboxService/Client/NetworkBootstrapInfo.swift @@ -14,19 +14,16 @@ // limitations under the License. //===----------------------------------------------------------------------===// -import ContainerXPC +import ContainerResource -/// AllocatedAttachment represents a network attachment that has been allocated for use -/// by a container and any additional relevant data needed for a sandbox to properly -/// configure networking on container bootstrap. -public struct AllocatedAttachment: Sendable { - public let attachment: Attachment - public let additionalData: XPCMessage? +/// Plugin info passed from the API server in the sandbox bootstrap message so the +/// runtime can connect to the correct network helper and configure the interface. +public struct NetworkBootstrapInfo: Codable, Sendable { + /// Plugin info identifying which network helper to contact and which interface + /// strategy the runtime should use. public let pluginInfo: NetworkPluginInfo - public init(attachment: Attachment, additionalData: XPCMessage?, pluginInfo: NetworkPluginInfo) { - self.attachment = attachment - self.additionalData = additionalData + public init(pluginInfo: NetworkPluginInfo) { self.pluginInfo = pluginInfo } } diff --git a/Sources/Services/ContainerSandboxService/Client/SandboxClient.swift b/Sources/Services/ContainerSandboxService/Client/SandboxClient.swift index f7bdfdad..23cbc946 100644 --- a/Sources/Services/ContainerSandboxService/Client/SandboxClient.swift +++ b/Sources/Services/ContainerSandboxService/Client/SandboxClient.swift @@ -79,7 +79,7 @@ public struct SandboxClient: Sendable { extension SandboxClient { public func bootstrap( stdio: [FileHandle?], - allocatedAttachments: [AllocatedAttachment], + networkBootstrapInfos: [NetworkBootstrapInfo], dynamicEnv: [String: String] = [:] ) async throws { let request = XPCMessage(route: SandboxRoutes.bootstrap.rawValue) @@ -104,7 +104,8 @@ extension SandboxClient { let dynamicEnv = try JSONEncoder().encode(dynamicEnv) request.set(key: SandboxKeys.dynamicEnv.rawValue, value: dynamicEnv) - try request.setAllocatedAttachments(allocatedAttachments) + let infosData = try JSONEncoder().encode(networkBootstrapInfos) + request.set(key: SandboxKeys.networkBootstrapInfos.rawValue, value: infosData) try await self.client.send(request) } catch { throw ContainerizationError( @@ -331,25 +332,10 @@ extension XPCMessage { return try JSONDecoder().decode(SandboxSnapshot.self, from: data) } - func setAllocatedAttachments(_ allocatedAttachments: [AllocatedAttachment]) throws { - let encoder = JSONEncoder() - let allocatedAttachmentsArray = xpc_array_create_empty() - for allocatedAttach in allocatedAttachments { - let xpcObject: xpc_object_t = xpc_dictionary_create_empty() - let networkXPC = XPCMessage(object: xpcObject) - - let attachmentEncoded = try encoder.encode(allocatedAttach.attachment) - networkXPC.set(key: SandboxKeys.networkAttachment.rawValue, value: attachmentEncoded) - - let pluginInfoEncoded = try encoder.encode(allocatedAttach.pluginInfo) - networkXPC.set(key: SandboxKeys.networkPluginInfo.rawValue, value: pluginInfoEncoded) - - if let additionalData = allocatedAttach.additionalData { - xpc_dictionary_set_value(networkXPC.underlying, SandboxKeys.networkAdditionalData.rawValue, additionalData.underlying) - } - - xpc_array_append_value(allocatedAttachmentsArray, networkXPC.underlying) + public func networkBootstrapInfos() throws -> [NetworkBootstrapInfo] { + guard let data = self.dataNoCopy(key: SandboxKeys.networkBootstrapInfos.rawValue) else { + throw ContainerizationError(.invalidArgument, message: "missing networkBootstrapInfos in bootstrap message") } - self.set(key: SandboxKeys.allocatedAttachments.rawValue, value: allocatedAttachmentsArray) + return try JSONDecoder().decode([NetworkBootstrapInfo].self, from: data) } } diff --git a/Sources/Services/ContainerSandboxService/Client/SandboxKeys.swift b/Sources/Services/ContainerSandboxService/Client/SandboxKeys.swift index cdf5ae1f..ef9719af 100644 --- a/Sources/Services/ContainerSandboxService/Client/SandboxKeys.swift +++ b/Sources/Services/ContainerSandboxService/Client/SandboxKeys.swift @@ -46,9 +46,6 @@ public enum SandboxKeys: String { /// Special-case environment variables recomputed on each container start case dynamicEnv - /// Network resource keys. - case allocatedAttachments - case networkAdditionalData - case networkAttachment - case networkPluginInfo + /// Per-network connection info passed to the runtime so it can allocate directly. + case networkBootstrapInfos } diff --git a/Sources/Services/ContainerSandboxService/Server/SandboxService.swift b/Sources/Services/ContainerSandboxService/Server/SandboxService.swift index bb6806b5..1383b8b5 100644 --- a/Sources/Services/ContainerSandboxService/Server/SandboxService.swift +++ b/Sources/Services/ContainerSandboxService/Server/SandboxService.swift @@ -14,7 +14,7 @@ // limitations under the License. //===----------------------------------------------------------------------===// -import ContainerAPIClient +import ContainerNetworkServiceClient import ContainerOS import ContainerPersistence import ContainerResource @@ -50,6 +50,7 @@ public actor SandboxService { private var state: State = .created private var processes: [String: ProcessInfo] = [:] private var socketForwarders: [SocketForwarderResult] = [] + private var networkSessions: [XPCClientSession] = [] private static let sshAuthSocketGuestPath = "/var/host-services/ssh-auth.sock" private static let sshAuthSocketEnvVar = "SSH_AUTH_SOCK" @@ -168,11 +169,53 @@ public actor SandboxService { logger: self.log ) - let allocatedAttachments = try message.getAllocatedAttachments() + let networkBootstrapInfos = try message.networkBootstrapInfos() + + var sessions: [XPCClientSession] = [] + var attachments: [Attachment] = [] + var interfaces: [Interface] = [] + do { + for (index, info) in networkBootstrapInfos.enumerated() { + let attachmentConfig = config.networks[index] + let client = ContainerNetworkServiceClient.NetworkClient(id: attachmentConfig.network, plugin: info.pluginInfo.plugin) + let session = client.connect() + sessions.append(session) + var (attachment, additionalData) = try await client.allocate( + hostname: attachmentConfig.options.hostname, + macAddress: attachmentConfig.options.macAddress, + on: session + ) + if let mtu = attachmentConfig.options.mtu { + attachment = Attachment( + network: attachment.network, + hostname: attachment.hostname, + ipv4Address: attachment.ipv4Address, + ipv4Gateway: attachment.ipv4Gateway, + ipv6Address: attachment.ipv6Address, + macAddress: attachment.macAddress, + mtu: mtu + ) + } + guard let iStrategy = self.interfaceStrategies[info.pluginInfo] else { + throw ContainerizationError( + .internalError, message: "no available interface strategy for network \(attachment.network), \(info.pluginInfo)") + } + let interface = try iStrategy.toInterface( + attachment: attachment, + interfaceIndex: index, + additionalData: additionalData + ) + attachments.append(attachment) + interfaces.append(interface) + } + } catch { + for session in sessions { session.close() } + throw error + } // Dynamically configure the DNS nameserver from a network if no explicit configuration if let dns = config.dns, dns.nameservers.isEmpty { - let defaultNameservers = try await self.getDefaultNameservers(allocatedAttachments: allocatedAttachments) + let defaultNameservers = self.getDefaultNameservers(from: attachments) if !defaultNameservers.isEmpty { config.dns = ContainerConfiguration.DNSConfiguration( nameservers: defaultNameservers, @@ -183,25 +226,6 @@ public actor SandboxService { } } - var attachments: [Attachment] = [] - var interfaces: [Interface] = [] - for index in 0.. [String] { - let networkClient = NetworkClient() - for allocatedAttach in allocatedAttachments { - let state = try await networkClient.get(id: allocatedAttach.attachment.network) - guard state.status.phase == "running", let gateway = state.status.ipv4Gateway else { - continue - } - return [gateway.description] + private nonisolated func getDefaultNameservers(from attachments: [Attachment]) -> [String] { + for attachment in attachments { + return [attachment.ipv4Gateway.description] } - return [] } @@ -1131,6 +1150,9 @@ public actor SandboxService { await self.stopSocketForwarders() + for session in networkSessions { session.close() } + networkSessions = [] + let status = exitStatus ?? ExitStatus(exitCode: 255) self.releaseWaiters(for: id, status: status) } @@ -1184,49 +1206,6 @@ extension XPCMessage { return dynamicEnv } - fileprivate func getAllocatedAttachments() throws -> [AllocatedAttachment] { - guard let attachmentArray = xpc_dictionary_get_value(self.underlying, SandboxKeys.allocatedAttachments.rawValue) else { - throw ContainerizationError(.invalidArgument, message: "missing allocatedAttachments array in message") - } - - var results = [AllocatedAttachment]() - let decoder = JSONDecoder() - - let arrayCount = xpc_array_get_count(attachmentArray) - - for i in 0..