container/Sources/ContainerCluster/Commands/ClusterCreate.swift

//===----------------------------------------------------------------------===//
// Copyright © 2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//

import ArgumentParser
import ContainerAPIClient
import ContainerLog
import ContainerPersistence
import ContainerResource
import ContainerizationError
import Darwin
import Foundation
import Logging
import TerminalProgress

public struct ClusterCreate: AsyncParsableCommand {
    public init() {}

    public static let configuration = CommandConfiguration(
        commandName: "create",
        abstract: "Create and start a local Kubernetes cluster and optional worker nodes"
    )

    @Option(name: .long, help: "Cluster name")
    var name: String = ClusterHelper.defaultName

    @Flag(name: [.customLong("rm"), .long], help: "Remove the cluster container after it stops")
    var remove: Bool = false

    @OptionGroup(title: "Resource options")
    var resourceFlags: Flags.Resource

    @OptionGroup(title: "Registry options")
    var registryFlags: Flags.Registry

    @OptionGroup(title: "Image fetch options")
    var imageFetchFlags: Flags.ImageFetch

    @Option(help: "Node image reference")
    var nodeImage: String = ClusterHelper.nodeImage

    @Option(name: .long, help: "Optional path to a CNI manifest to apply, or \"NONE\" to skip installing a CNI.")
    var cni: String?

    @Option(name: .long, help: "Number of worker nodes to create")
    var workers: UInt = 0

    /// Task count is `4 + workers` as an `Int`, so keep `workers` far enough below `Int.max` to avoid overflow.
    static let maxWorkers: UInt = 1024

    public func validate() throws {
        guard workers <= Self.maxWorkers else {
            throw ValidationError("--workers must be at most \(Self.maxWorkers)")
        }
    }

    public func run() async throws {
        LoggingSystem.bootstrap { _ in StderrLogHandler() }
        let log = Logger(label: ClusterHelper.pluginName)

        guard ManagedContainer.nameValid(name) else {
            throw ContainerizationError(.invalidArgument, message: "cluster name \(name) is not a valid container ID")
        }

        let skipCNI = cni?.uppercased() == ClusterHelper.noCNIName
        if let cni, !skipCNI {
            guard FileManager.default.fileExists(atPath: cni) else {
                throw ContainerizationError(.invalidArgument, message: "CNI manifest not found at \(cni)")
            }
        }

        // Fail before provisioning the node VM.
        _ = try ClusterHelper.kubernetesVersion(nodeImage: nodeImage)

        let isTTY = isatty(FileHandle.standardError.fileDescriptor) == 1
        // TotalTasks include:
        // start, kubeadm init, one per worker, wait, write kubeconfig.
        let progressConfig = try ProgressConfig(
            showSpinner: isTTY,
            showTasks: true,
            showItems: true,
            ignoreSmallSize: true,
            totalTasks: 4 + Int(workers),
            clearOnFinish: isTTY,
            outputMode: isTTY ? .ansi : .plain
        )

        let progress = ProgressBar(config: progressConfig)
        defer { progress.finish() }
        progress.start()

        let containerSystemConfig: ContainerSystemConfig = try await ConfigurationLoader.load()
        let fqdn = ClusterHelper.fqdn(for: name, domain: containerSystemConfig.dns.domain)

        let provisioner = try LinuxNodeProvisioner(
            clusterName: name,
            roles: workers == 0 ? [StandardRoles.controlPlane, StandardRoles.worker] : [StandardRoles.controlPlane],
            nodeImage: nodeImage,
            cpus: resourceFlags.cpus,
            memory: resourceFlags.memory,
            registryScheme: registryFlags.scheme,
            maxConcurrentDownloads: imageFetchFlags.maxConcurrentDownloads,
            remove: remove,
            fqdn: fqdn
        )

        progress.set(description: "Starting cluster")
        try await provisioner.provision(name: name, log: log)

        let client = ContainerClient()
        var workerNames: [String] = []
        do {
            let vmIP = try await provisioner.address(name: name, log: log)
            var sans = ["127.0.0.1"]
            if let fqdn { sans.append(contentsOf: [vmIP, fqdn]) }

            progress.set(description: "Running kubeadm init")
            try await ClusterHelper.prepareNode(nodeID: name, client: client, log: log)
            try await ClusterHelper.bootstrapControlPlane(
                nodeID: name, nodeImage: nodeImage, apiServerSANs: sans, advertiseAddress: vmIP,
                schedulable: provisioner.roles.contains(StandardRoles.worker),
                cniManifestPath: cni,
                client: client, log: log)

            if workers > 0 {
                let (token, caCertHash) = try await ClusterHelper.createJoinToken(nodeID: name, client: client)
                let controlPlaneEndpoint = "\(vmIP):\(ClusterHelper.clusterContainerPort)"
                for i in 1...workers {
                    let workerName = "\(name)-worker-\(i)"
                    progress.set(description: "Joining worker \(workerName)")
                    let workerProvisioner = try LinuxNodeProvisioner(
                        clusterName: name,
                        roles: [StandardRoles.worker],
                        nodeImage: nodeImage,
                        cpus: resourceFlags.cpus,
                        memory: resourceFlags.memory,
                        registryScheme: registryFlags.scheme,
                        maxConcurrentDownloads: imageFetchFlags.maxConcurrentDownloads,
                        remove: remove
                    )
                    workerNames.append(workerName)
                    try await workerProvisioner.provision(name: workerName, log: log)
                    try await workerProvisioner.join(
                        name: workerName, controlPlaneEndpoint: controlPlaneEndpoint,
                        token: token, caCertHash: caCertHash, log: log)
                    if skipCNI {
                        try await workerProvisioner.waitForRegistered(name: workerName, log: log)
                    } else {
                        try await workerProvisioner.waitForReady(name: workerName, log: log)
                    }
                }
            }

            if skipCNI {
                progress.set(description: "Waiting for API server")
                try await ClusterHelper.waitForAPIServer(containerId: name, client: client, log: log)
            } else {
                progress.set(description: "Waiting for cluster to be ready")
                try await ClusterHelper.waitForReady(containerId: name, client: client, log: log)
            }

            progress.set(description: "Writing kubeconfig")
            let rawConfig = try await ClusterHelper.fetchConfig(containerId: name, client: client, log: log)
            let kubeConfig = try await ClusterHelper.transformConfig(rawConfig, containerId: name, fqdn: fqdn, client: client)
            do {
                try ClusterHelper.mergeConfig(kubeConfig, containerId: name, setCurrentContext: true, log: log)
            } catch {
                log.warning("failed to write kubeconfig", metadata: ["name": "\(name)", "error": "\(error)"])
                log.info("cluster is running; use 'container cluster write-config --name \(name)' to write the kubeconfig")
            }
        } catch {
            for workerName in workerNames {
                await Self.teardownIfOwned(workerName, client: client, provisioner: provisioner, log: log)
            }
            await Self.teardownIfOwned(name, client: client, provisioner: provisioner, log: log)
            try? ClusterHelper.removeConfig(containerId: name, log: log)
            throw error
        }

        progress.finish()
        print(name)
    }

    /// Tears down `containerName` only if it's a container this plugin created
    private static func teardownIfOwned(
        _ containerName: String, client: ContainerClient, provisioner: LinuxNodeProvisioner, log: Logger
    ) async {
        guard let container = try? await client.get(id: containerName) else { return }
        guard container.configuration.labels[ResourceLabelKeys.plugin] == ClusterHelper.pluginName else {
            log.warning(
                "container exists but is not owned by this plugin, refusing teardown",
                metadata: ["name": "\(containerName)"])
            return
        }
        try? await provisioner.teardown(name: containerName, log: log)
    }
}