Remove IP allocation from API server. (#1545)

- Closes #1318. - Closes #1378. - Reduces the complexity and coupling for IP allocation. - Runtimes connect to networks for the life of the running container. The runtime shuts down on connection loss. - Networks automatically deallocate a runtime's IP address and hostname record on connection loss. - Removes AllocatedAttachment as this is no longer necessary. The `bootstrap()` XPC now takes a `NetworkBootstrapInfo` array which parallels the attachments in the bundle config and provides the network plugin attributes needed to create VM network interface configurations. ## Type of Change - [x] Bug fix - [ ] New feature - [ ] Breaking change - [ ] Documentation update ## Motivation and Context Simplify IP allocation and make deallocation more reliable. ## Testing - [x] Tested locally - [ ] Added/updated tests - [ ] Added/updated docs
2026-09-22 23:55:34 +00:00 · 2026-05-12 16:34:06 -07:00
parent 8a46476341
commit b466959644
8 changed files with 90 additions and 183 deletions
@@ -358,7 +358,7 @@ let package = Package(
                .product(name: "ContainerizationExtras", package: "containerization"),
                .product(name: "ContainerizationOS", package: "containerization"),
                .product(name: "ArgumentParser", package: "swift-argument-parser"),
-                "ContainerAPIClient",
+                "ContainerNetworkServiceClient",
                "ContainerOS",
                "ContainerPersistence",
                "ContainerResource",
@@ -34,9 +34,7 @@ import SystemPackage
 public actor ContainersService {
    struct ContainerState {
        var snapshot: ContainerSnapshot
-        var client: SandboxClient?
-        var allocatedAttachments: [AllocatedAttachment]
-        var networkSessions: [XPCClientSession]
+        var client: SandboxClient? = nil

        func getClient() throws -> SandboxClient {
            guard let client else {
@@ -133,8 +131,6 @@ public actor ContainersService {
                        networks: [],
                        startedDate: nil
                    ),
-                    allocatedAttachments: [],
-                    networkSessions: []
                )
                results[config.id] = state
                guard runtimePlugins.first(where: { $0.name == config.runtimeHandler }) != nil else {
@@ -396,7 +392,7 @@ public actor ContainersService {
                    networks: [],
                    startedDate: nil
                )
-                await self.setContainerState(configuration.id, ContainerState(snapshot: snapshot, allocatedAttachments: [], networkSessions: []), context: context)
+                await self.setContainerState(configuration.id, ContainerState(snapshot: snapshot), context: context)
            } catch {
                throw error
            }
@@ -436,41 +432,15 @@ public actor ContainersService {
            let path = self.containerRoot.appendingPathComponent(id)
            let (config, _) = try Self.getContainerConfiguration(at: path)

-            var allocatedAttachments = [AllocatedAttachment]()
-            var networkSessions = [XPCClientSession]()
-            do {
-                for n in config.networks {
-                    guard
-                        let (allocatedAttach, session) = try await self.networksService?.allocate(
-                            id: n.network,
-                            hostname: n.options.hostname,
-                            macAddress: n.options.macAddress
-                        )
-                    else {
-                        throw ContainerizationError(.internalError, message: "failed to allocate a network")
-                    }
-
-                    var finalAttach = allocatedAttach
-                    if let mtu = n.options.mtu {
-                        let a = allocatedAttach.attachment
-                        finalAttach = AllocatedAttachment(
-                            attachment: Attachment(
-                                network: a.network,
-                                hostname: a.hostname,
-                                ipv4Address: a.ipv4Address,
-                                ipv4Gateway: a.ipv4Gateway,
-                                ipv6Address: a.ipv6Address,
-                                macAddress: a.macAddress,
-                                mtu: mtu
-                            ),
-                            additionalData: allocatedAttach.additionalData,
-                            pluginInfo: allocatedAttach.pluginInfo
-                        )
-                    }
-                    allocatedAttachments.append(finalAttach)
-                    networkSessions.append(session)
+            var networkBootstrapInfos = [NetworkBootstrapInfo]()
+            for n in config.networks {
+                guard let pluginInfo = try await self.networksService?.pluginInfo(id: n.network) else {
+                    throw ContainerizationError(.internalError, message: "failed to get plugin info for network \(n.network)")
                }
+                networkBootstrapInfos.append(NetworkBootstrapInfo(pluginInfo: pluginInfo))
+            }

+            do {
                try Self.registerService(
                    plugin: self.runtimePlugins.first { $0.name == config.runtimeHandler }!,
                    loader: self.pluginLoader,
@@ -484,7 +454,7 @@ public actor ContainersService {
                    id: id,
                    runtime: runtime
                )
-                try await sandboxClient.bootstrap(stdio: stdio, allocatedAttachments: allocatedAttachments, dynamicEnv: dynamicEnv)
+                try await sandboxClient.bootstrap(stdio: stdio, networkBootstrapInfos: networkBootstrapInfos, dynamicEnv: dynamicEnv)

                try await self.exitMonitor.registerProcess(
                    id: id,
@@ -492,14 +462,8 @@ public actor ContainersService {
                )

                state.client = sandboxClient
-                state.allocatedAttachments = allocatedAttachments
-                state.networkSessions = networkSessions
                await self.setContainerState(id, state, context: context)
            } catch {
-                for session in networkSessions {
-                    session.close()
-                }
-
                let label = Self.fullLaunchdServiceLabel(
                    runtimeName: config.runtimeHandler,
                    instanceId: id
@@ -994,17 +958,9 @@ public actor ContainersService {
                ])
        }

-        // Close network sessions — the network helper auto-releases allocations on disconnect.
-        self.log.info("closing network sessions", metadata: ["id": "\(id)"])
-        for session in state.networkSessions {
-            session.close()
-        }
-
        state.snapshot.status = .stopped
        state.snapshot.networks = []
        state.client = nil
-        state.allocatedAttachments = []
-        state.networkSessions = []
        await self.setContainerState(id, state, context: context)

        let options = try getContainerCreationOptions(id: id)
@@ -19,7 +19,6 @@ import ContainerNetworkServiceClient
 import ContainerPersistence
 import ContainerPlugin
 import ContainerResource
-import ContainerXPC
 import Containerization
 import ContainerizationError
 import ContainerizationExtras
@@ -380,26 +379,14 @@ public actor NetworksService {
        }
    }

-    public func allocate(id: String, hostname: String, macAddress: MACAddress?) async throws -> (AllocatedAttachment, XPCClientSession) {
+    public func pluginInfo(id: String) throws -> NetworkPluginInfo {
        guard let serviceState = serviceStates[id] else {
            throw ContainerizationError(.notFound, message: "no network for id \(id)")
        }
        guard let pluginInfo = serviceState.networkState.pluginInfo else {
            throw ContainerizationError(.internalError, message: "network \(id) missing plugin information")
        }
-        let session = serviceState.client.connect()
-        do {
-            let (attach, additionalData) = try await serviceState.client.allocate(hostname: hostname, macAddress: macAddress, on: session)
-            let alloc = AllocatedAttachment(
-                attachment: attach,
-                additionalData: additionalData,
-                pluginInfo: pluginInfo
-            )
-            return (alloc, session)
-        } catch {
-            session.close()
-            throw error
-        }
+        return pluginInfo
    }

    private static func getClient(configuration: NetworkConfiguration) throws -> ContainerNetworkServiceClient.NetworkClient {
@@ -27,7 +27,7 @@ public actor NetworkService: Sendable {
    private let log: Logger
    private var allocator: AttachmentAllocator
    private var macAddresses: [UInt32: MACAddress]
-    private var allocationsBySession: [XPCServerSession: [(hostname: String, index: UInt32)]] = [:]
+    private var allocationsBySession: [XPCServerSession: [(hostname: String, index: UInt32)]]

    /// Set up a network service for the specified network.
    public init(
@@ -42,10 +42,11 @@ public actor NetworkService: Sendable {
        let subnet = status.ipv4Subnet

        let size = Int(subnet.upper.value - subnet.lower.value - 3)
-        self.allocator = try AttachmentAllocator(lower: subnet.lower.value + 2, size: size)
-        self.macAddresses = [:]
        self.network = network
        self.log = log
+        self.allocator = try AttachmentAllocator(lower: subnet.lower.value + 2, size: size)
+        self.macAddresses = [:]
+        self.allocationsBySession = [:]
    }

    @Sendable
@@ -14,19 +14,16 @@
 // limitations under the License.
 //===----------------------------------------------------------------------===//

-import ContainerXPC
+import ContainerResource

-/// AllocatedAttachment represents a network attachment that has been allocated for use
-/// by a container and any additional relevant data needed for a sandbox to properly
-/// configure networking on container bootstrap.
-public struct AllocatedAttachment: Sendable {
-    public let attachment: Attachment
-    public let additionalData: XPCMessage?
+/// Plugin info passed from the API server in the sandbox bootstrap message so the
+/// runtime can connect to the correct network helper and configure the interface.
+public struct NetworkBootstrapInfo: Codable, Sendable {
+    /// Plugin info identifying which network helper to contact and which interface
+    /// strategy the runtime should use.
    public let pluginInfo: NetworkPluginInfo

-    public init(attachment: Attachment, additionalData: XPCMessage?, pluginInfo: NetworkPluginInfo) {
-        self.attachment = attachment
-        self.additionalData = additionalData
+    public init(pluginInfo: NetworkPluginInfo) {
        self.pluginInfo = pluginInfo
    }
 }
@@ -79,7 +79,7 @@ public struct SandboxClient: Sendable {
 extension SandboxClient {
    public func bootstrap(
        stdio: [FileHandle?],
-        allocatedAttachments: [AllocatedAttachment],
+        networkBootstrapInfos: [NetworkBootstrapInfo],
        dynamicEnv: [String: String] = [:]
    ) async throws {
        let request = XPCMessage(route: SandboxRoutes.bootstrap.rawValue)
@@ -104,7 +104,8 @@ extension SandboxClient {
            let dynamicEnv = try JSONEncoder().encode(dynamicEnv)
            request.set(key: SandboxKeys.dynamicEnv.rawValue, value: dynamicEnv)

-            try request.setAllocatedAttachments(allocatedAttachments)
+            let infosData = try JSONEncoder().encode(networkBootstrapInfos)
+            request.set(key: SandboxKeys.networkBootstrapInfos.rawValue, value: infosData)
            try await self.client.send(request)
        } catch {
            throw ContainerizationError(
@@ -331,25 +332,10 @@ extension XPCMessage {
        return try JSONDecoder().decode(SandboxSnapshot.self, from: data)
    }

-    func setAllocatedAttachments(_ allocatedAttachments: [AllocatedAttachment]) throws {
-        let encoder = JSONEncoder()
-        let allocatedAttachmentsArray = xpc_array_create_empty()
-        for allocatedAttach in allocatedAttachments {
-            let xpcObject: xpc_object_t = xpc_dictionary_create_empty()
-            let networkXPC = XPCMessage(object: xpcObject)
-
-            let attachmentEncoded = try encoder.encode(allocatedAttach.attachment)
-            networkXPC.set(key: SandboxKeys.networkAttachment.rawValue, value: attachmentEncoded)
-
-            let pluginInfoEncoded = try encoder.encode(allocatedAttach.pluginInfo)
-            networkXPC.set(key: SandboxKeys.networkPluginInfo.rawValue, value: pluginInfoEncoded)
-
-            if let additionalData = allocatedAttach.additionalData {
-                xpc_dictionary_set_value(networkXPC.underlying, SandboxKeys.networkAdditionalData.rawValue, additionalData.underlying)
-            }
-
-            xpc_array_append_value(allocatedAttachmentsArray, networkXPC.underlying)
+    public func networkBootstrapInfos() throws -> [NetworkBootstrapInfo] {
+        guard let data = self.dataNoCopy(key: SandboxKeys.networkBootstrapInfos.rawValue) else {
+            throw ContainerizationError(.invalidArgument, message: "missing networkBootstrapInfos in bootstrap message")
        }
-        self.set(key: SandboxKeys.allocatedAttachments.rawValue, value: allocatedAttachmentsArray)
+        return try JSONDecoder().decode([NetworkBootstrapInfo].self, from: data)
    }
 }
@@ -46,9 +46,6 @@ public enum SandboxKeys: String {
    /// Special-case environment variables recomputed on each container start
    case dynamicEnv

-    /// Network resource keys.
-    case allocatedAttachments
-    case networkAdditionalData
-    case networkAttachment
-    case networkPluginInfo
+    /// Per-network connection info passed to the runtime so it can allocate directly.
+    case networkBootstrapInfos
 }
@@ -14,7 +14,7 @@
 // limitations under the License.
 //===----------------------------------------------------------------------===//

-import ContainerAPIClient
+import ContainerNetworkServiceClient
 import ContainerOS
 import ContainerPersistence
 import ContainerResource
@@ -50,6 +50,7 @@ public actor SandboxService {
    private var state: State = .created
    private var processes: [String: ProcessInfo] = [:]
    private var socketForwarders: [SocketForwarderResult] = []
+    private var networkSessions: [XPCClientSession] = []

    private static let sshAuthSocketGuestPath = "/var/host-services/ssh-auth.sock"
    private static let sshAuthSocketEnvVar = "SSH_AUTH_SOCK"
@@ -168,11 +169,53 @@ public actor SandboxService {
                logger: self.log
            )

-            let allocatedAttachments = try message.getAllocatedAttachments()
+            let networkBootstrapInfos = try message.networkBootstrapInfos()
+
+            var sessions: [XPCClientSession] = []
+            var attachments: [Attachment] = []
+            var interfaces: [Interface] = []
+            do {
+                for (index, info) in networkBootstrapInfos.enumerated() {
+                    let attachmentConfig = config.networks[index]
+                    let client = ContainerNetworkServiceClient.NetworkClient(id: attachmentConfig.network, plugin: info.pluginInfo.plugin)
+                    let session = client.connect()
+                    sessions.append(session)
+                    var (attachment, additionalData) = try await client.allocate(
+                        hostname: attachmentConfig.options.hostname,
+                        macAddress: attachmentConfig.options.macAddress,
+                        on: session
+                    )
+                    if let mtu = attachmentConfig.options.mtu {
+                        attachment = Attachment(
+                            network: attachment.network,
+                            hostname: attachment.hostname,
+                            ipv4Address: attachment.ipv4Address,
+                            ipv4Gateway: attachment.ipv4Gateway,
+                            ipv6Address: attachment.ipv6Address,
+                            macAddress: attachment.macAddress,
+                            mtu: mtu
+                        )
+                    }
+                    guard let iStrategy = self.interfaceStrategies[info.pluginInfo] else {
+                        throw ContainerizationError(
+                            .internalError, message: "no available interface strategy for network \(attachment.network), \(info.pluginInfo)")
+                    }
+                    let interface = try iStrategy.toInterface(
+                        attachment: attachment,
+                        interfaceIndex: index,
+                        additionalData: additionalData
+                    )
+                    attachments.append(attachment)
+                    interfaces.append(interface)
+                }
+            } catch {
+                for session in sessions { session.close() }
+                throw error
+            }

            // Dynamically configure the DNS nameserver from a network if no explicit configuration
            if let dns = config.dns, dns.nameservers.isEmpty {
-                let defaultNameservers = try await self.getDefaultNameservers(allocatedAttachments: allocatedAttachments)
+                let defaultNameservers = self.getDefaultNameservers(from: attachments)
                if !defaultNameservers.isEmpty {
                    config.dns = ContainerConfiguration.DNSConfiguration(
                        nameservers: defaultNameservers,
@@ -183,25 +226,6 @@ public actor SandboxService {
                }
            }

-            var attachments: [Attachment] = []
-            var interfaces: [Interface] = []
-            for index in 0..<allocatedAttachments.count {
-                let allocatedAttach = allocatedAttachments[index]
-                attachments.append(allocatedAttach.attachment)
-
-                guard let iStrategy = self.interfaceStrategies[allocatedAttach.pluginInfo] else {
-                    throw ContainerizationError(
-                        .internalError, message: "no available interface strategy for network \(allocatedAttach.attachment.network), \(allocatedAttach.pluginInfo)")
-                }
-
-                let interface = try iStrategy.toInterface(
-                    attachment: allocatedAttach.attachment,
-                    interfaceIndex: index,
-                    additionalData: allocatedAttach.additionalData
-                )
-                interfaces.append(interface)
-            }
-
            let stdio = message.stdio()
            let containerLog = try FileHandle(forWritingTo: bundle.containerLog)
            let stdout = {
@@ -256,6 +280,7 @@ public actor SandboxService {
                io: (in: stdin, out: stdout, err: stderr)
            )
            await self.setContainer(ctrInfo)
+            await self.setNetworkSessions(sessions)

            do {
                try await container.create()
@@ -922,16 +947,10 @@ public actor SandboxService {
        try Self.configureInitialProcess(czConfig: &czConfig, config: config)
    }

-    private func getDefaultNameservers(allocatedAttachments: [AllocatedAttachment]) async throws -> [String] {
-        let networkClient = NetworkClient()
-        for allocatedAttach in allocatedAttachments {
-            let state = try await networkClient.get(id: allocatedAttach.attachment.network)
-            guard state.status.phase == "running", let gateway = state.status.ipv4Gateway else {
-                continue
-            }
-            return [gateway.description]
+    private nonisolated func getDefaultNameservers(from attachments: [Attachment]) -> [String] {
+        for attachment in attachments {
+            return [attachment.ipv4Gateway.description]
        }
-
        return []
    }

@@ -1131,6 +1150,9 @@ public actor SandboxService {

        await self.stopSocketForwarders()

+        for session in networkSessions { session.close() }
+        networkSessions = []
+
        let status = exitStatus ?? ExitStatus(exitCode: 255)
        self.releaseWaiters(for: id, status: status)
    }
@@ -1184,49 +1206,6 @@ extension XPCMessage {
        return dynamicEnv
    }

-    fileprivate func getAllocatedAttachments() throws -> [AllocatedAttachment] {
-        guard let attachmentArray = xpc_dictionary_get_value(self.underlying, SandboxKeys.allocatedAttachments.rawValue) else {
-            throw ContainerizationError(.invalidArgument, message: "missing allocatedAttachments array in message")
-        }
-
-        var results = [AllocatedAttachment]()
-        let decoder = JSONDecoder()
-
-        let arrayCount = xpc_array_get_count(attachmentArray)
-
-        for i in 0..<arrayCount {
-            guard let allocatedAttach = xpc_array_get_dictionary(attachmentArray, i) else {
-                throw ContainerizationError(.invalidArgument, message: "invalid allocated attachment at index \(i)")
-            }
-
-            let allocatedAttachXPC = XPCMessage(object: allocatedAttach)
-
-            let attachmentData = allocatedAttachXPC.dataNoCopy(key: SandboxKeys.networkAttachment.rawValue)
-            let pluginInfoData = allocatedAttachXPC.dataNoCopy(key: SandboxKeys.networkPluginInfo.rawValue)
-
-            guard let attachmentData = attachmentData, let pluginInfoData = pluginInfoData else {
-                throw ContainerizationError(.invalidArgument, message: "must have attachment and plugin information for network")
-            }
-
-            let attachment = try decoder.decode(Attachment.self, from: attachmentData)
-            let pluginInfo = try decoder.decode(NetworkPluginInfo.self, from: pluginInfoData)
-
-            let additionalDataXPC: XPCMessage? = {
-                if let rawData = xpc_dictionary_get_dictionary(allocatedAttachXPC.underlying, SandboxKeys.networkAdditionalData.rawValue) {
-                    return XPCMessage(object: rawData)
-                }
-                return nil
-            }()
-
-            results.append(
-                AllocatedAttachment(
-                    attachment: attachment,
-                    additionalData: additionalDataXPC,
-                    pluginInfo: pluginInfo
-                ))
-        }
-        return results
-    }
 }

 extension ContainerResource.Bundle {
@@ -1399,6 +1378,10 @@ extension SandboxService {
        self.container = info
    }

+    private func setNetworkSessions(_ sessions: [XPCClientSession]) {
+        self.networkSessions = sessions
+    }
+
    private func addNewProcess(_ id: String, _ config: ProcessConfiguration, _ io: [FileHandle?]) throws {
        guard self.processes[id] == nil else {
            throw ContainerizationError(.invalidArgument, message: "process \(id) already exists")