diff --git a/Sources/ContainerK8s/Commands/K8sCreate.swift b/Sources/ContainerK8s/Commands/K8sCreate.swift index 80f537a8..ba6e0a2e 100644 --- a/Sources/ContainerK8s/Commands/K8sCreate.swift +++ b/Sources/ContainerK8s/Commands/K8sCreate.swift @@ -51,7 +51,7 @@ public struct K8sCreate: AsyncParsableCommand { @Option(help: "Node image reference (default: \(K8sHelper.nodeImage))") var nodeImage: String = K8sHelper.nodeImage - @Option(name: .long, help: "Optional path to a CNI manifest to apply.") + @Option(name: .long, help: "Optional path to a CNI manifest to apply, or \"NONE\" to skip installing a CNI.") var cni: String? public func run() async throws { @@ -62,7 +62,8 @@ public struct K8sCreate: AsyncParsableCommand { throw ContainerizationError(.invalidArgument, message: "cluster name \(name) is not a valid container ID") } - if let cni { + let skipCNI = cni?.uppercased() == K8sHelper.noCNIName + if let cni, !skipCNI { guard FileManager.default.fileExists(atPath: cni) else { throw ContainerizationError(.invalidArgument, message: "CNI manifest not found at \(cni)") } @@ -118,13 +119,18 @@ public struct K8sCreate: AsyncParsableCommand { cniManifestPath: cni, client: client, log: log) - progress.set(description: "Waiting for cluster to be ready") - try await K8sHelper.waitForReady(containerId: name, client: client, log: log) + if skipCNI { + progress.set(description: "Waiting for API server") + try await K8sHelper.waitForAPIServer(containerId: name, client: client, log: log) + } else { + progress.set(description: "Waiting for cluster to be ready") + try await K8sHelper.waitForReady(containerId: name, client: client, log: log) + } progress.set(description: "Writing kubeconfig") + let rawConfig = try await K8sHelper.fetchConfig(containerId: name, client: client, log: log) + let kubeConfig = try await K8sHelper.transformConfig(rawConfig, containerId: name, fqdn: fqdn, client: client) do { - let rawConfig = try await K8sHelper.fetchConfig(containerId: name, client: client, log: log) - let kubeConfig = try await K8sHelper.transformConfig(rawConfig, containerId: name, fqdn: fqdn, client: client) try K8sHelper.mergeConfig(kubeConfig, containerId: name, setCurrentContext: true, log: log) } catch { log.warning("failed to write kubeconfig", metadata: ["name": "\(name)", "error": "\(error)"]) diff --git a/Sources/ContainerK8s/K8sHelper.swift b/Sources/ContainerK8s/K8sHelper.swift index c999abfe..7bfd622f 100644 --- a/Sources/ContainerK8s/K8sHelper.swift +++ b/Sources/ContainerK8s/K8sHelper.swift @@ -44,6 +44,8 @@ public struct K8sHelper { public static let ignorePreflightErrors = "Swap,SystemVerification,FileContent--proc-sys-net-bridge-bridge-nf-call-iptables" static let podSubnet = "10.244.0.0/16" + /// Sentinel value for `--cni` (compared case-insensitively) that skips installing a CNI entirely. + static let noCNIName = "NONE" // kubeadm default service subnet; must stay in sync if ClusterConfiguration.serviceSubnet is ever set. static let serviceSubnet = "10.96.0.0/12" diff --git a/Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift b/Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift index 15512dc6..72079451 100644 --- a/Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift +++ b/Sources/ContainerK8s/Support/K8sHelper+Bootstrap.swift @@ -68,19 +68,26 @@ extension K8sHelper { if schedulable { log.info("Removing control-plane taint for single-node scheduling", metadata: ["node": "\(nodeID)"]) - _ = try await runProbe( + let taintCode = try await runProbe( client: client, containerId: nodeID, arguments: ["taint", "nodes", "--all", "node-role.kubernetes.io/control-plane-"]) + guard taintCode == 0 else { + throw ContainerizationError(.internalError, message: "failed to remove control-plane taint on \(nodeID)") + } } - log.info("Applying CNI manifest", metadata: ["node": "\(nodeID)"]) - let manifest = try await loadCNIManifest(path: cniManifestPath, log: log) - let apply = "\(kubeconfigEnv) kubectl apply -f - <<'EOF'\n\(manifest)\nEOF" - r = try await execCapture( - containerId: nodeID, executable: "/bin/sh", - arguments: ["-c", apply], client: client) - guard r.code == 0 else { - throw ContainerizationError(.internalError, message: "apply CNI failed on \(nodeID): \(r.output)") + if cniManifestPath?.uppercased() == noCNIName { + log.info("Skipping CNI installation", metadata: ["node": "\(nodeID)"]) + } else { + log.info("Applying CNI manifest", metadata: ["node": "\(nodeID)"]) + let manifest = try await loadCNIManifest(path: cniManifestPath, log: log) + let apply = "\(kubeconfigEnv) kubectl apply -f - <<'EOF'\n\(manifest)\nEOF" + r = try await execCapture( + containerId: nodeID, executable: "/bin/sh", + arguments: ["-c", apply], client: client) + guard r.code == 0 else { + throw ContainerizationError(.internalError, message: "apply CNI failed on \(nodeID): \(r.output)") + } } } diff --git a/Sources/ContainerK8s/Support/K8sHelper+Readiness.swift b/Sources/ContainerK8s/Support/K8sHelper+Readiness.swift index c6486544..e2dd3915 100644 --- a/Sources/ContainerK8s/Support/K8sHelper+Readiness.swift +++ b/Sources/ContainerK8s/Support/K8sHelper+Readiness.swift @@ -60,7 +60,35 @@ extension K8sHelper { } } + /// Wait until the API server is serving. This is the one readiness signal + /// that does not depend on a CNI being installed. + static func waitForAPIServer(containerId: String, client: ContainerClient, log: Logger) async throws { + let timeout = 60 + log.info("Waiting for API server to become available") + for attempt in 1...timeout { + let code: Int32 + do { + code = try await runProbe( + client: client, containerId: containerId, + arguments: ["get", "--raw", "/readyz", "--request-timeout=2s"]) + } catch { + throw ContainerizationError( + .internalError, message: "k8s cluster \(containerId) stopped unexpectedly during startup: \(error)") + } + if code == 0 { return } + if attempt == timeout { + log.info("check control-plane logs with 'container logs \(containerId)'") + throw ContainerizationError( + .timeout, + message: "k8s cluster \(containerId) API server did not become available within \(timeout * 2)s") + } + try await Task.sleep(for: .seconds(2)) + } + } + static func waitForReady(containerId: String, client: ContainerClient, log: Logger) async throws { + try await waitForAPIServer(containerId: containerId, client: client, log: log) + let nodeReadyTimeout = 180 let podReadyTimeout = 300 diff --git a/Tests/IntegrationTests/K8s/ContainerFixture+K8sHelpers.swift b/Tests/IntegrationTests/K8s/ContainerFixture+K8sHelpers.swift new file mode 100644 index 00000000..35ce88e7 --- /dev/null +++ b/Tests/IntegrationTests/K8s/ContainerFixture+K8sHelpers.swift @@ -0,0 +1,34 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerTestSupport +import Foundation + +extension ContainerFixture { + @discardableResult + func kubectl(node: String, args: [String]) throws -> (output: String, status: Int32) { + print("[k8s] kubectl \(args.joined(separator: " ")) (node: \(node))") + let result = try self.run(["exec", node, "kubectl"] + args) + print("[k8s] kubectl exit=\(result.status) output=\(result.output.prefix(120).trimmingCharacters(in: .whitespacesAndNewlines))") + let filteredStderr = result.error.components(separatedBy: "\n") + .filter { !$0.contains("Warning! Running debug build") && !$0.isEmpty } + .joined(separator: "\n") + if !filteredStderr.isEmpty { + print("[k8s] kubectl stderr: \(filteredStderr.prefix(300))") + } + return (result.output, result.status) + } +} diff --git a/Tests/IntegrationTests/K8s/TestK8sCNISerial.swift b/Tests/IntegrationTests/K8s/TestK8sCNISerial.swift new file mode 100644 index 00000000..e74e2629 --- /dev/null +++ b/Tests/IntegrationTests/K8s/TestK8sCNISerial.swift @@ -0,0 +1,52 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerTestSupport +import Foundation +import Testing + +@Suite(.serialized) +struct TestK8sCNISerial { + + @Test func testCreateWithCNINoneSkipsCNIInstallation() async throws { + try await ContainerFixture.with { f in + let name = "k8s-\(f.testID)" + f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) } + + try f.restoreWarmupImage(.kindestNodeV1_35_5) + print("[k8s-cni] k8s create --name \(name) --cni NONE") + let result = try f.run(["k8s", "create", "--name", name, "--cni", "NONE"]) + print("[k8s-cni] k8s create exit=\(result.status)") + if result.status != 0 { + print("[k8s-cni] k8s create stderr: \(result.error)") + f.dumpNodeDiagnostics(node: name) + } + + try result.check() + #expect(result.output.contains(name)) + #expect(try f.getContainerStatus(name) == "running") + + // No CNI manifest was applied, so kube-system has no CNI daemonset and the node never reaches Ready. + let (podsOutput, podsStatus) = try f.kubectl(node: name, args: ["get", "pods", "-n", "kube-system", "--no-headers"]) + #expect(podsStatus == 0) + #expect(!podsOutput.lowercased().contains("kindnet")) + + let (nodesOutput, nodesStatus) = try f.kubectl(node: name, args: ["get", "nodes", "--no-headers"]) + #expect(nodesStatus == 0) + #expect(nodesOutput.contains("NotReady")) + } + } +} diff --git a/Tests/IntegrationTests/K8s/TestK8sNetworkingSerial.swift b/Tests/IntegrationTests/K8s/TestK8sNetworkingSerial.swift index 1325f41e..34c17909 100644 --- a/Tests/IntegrationTests/K8s/TestK8sNetworkingSerial.swift +++ b/Tests/IntegrationTests/K8s/TestK8sNetworkingSerial.swift @@ -37,31 +37,17 @@ struct TestK8sNetworkingSerial { print("=== END ENV DUMP ===") } - @discardableResult - private func kubectl(_ f: ContainerFixture, node: String, args: [String]) throws -> (output: String, status: Int32) { - print("[k8s-net] kubectl \(args.joined(separator: " ")) (node: \(node))") - let result = try f.run(["exec", node, "kubectl"] + args) - print("[k8s-net] kubectl exit=\(result.status) output=\(result.output.prefix(120).trimmingCharacters(in: .whitespacesAndNewlines))") - let filteredStderr = result.error.components(separatedBy: "\n") - .filter { !$0.contains("Warning! Running debug build") && !$0.isEmpty } - .joined(separator: "\n") - if !filteredStderr.isEmpty { - print("[k8s-net] kubectl stderr: \(filteredStderr.prefix(300))") - } - return (result.output, result.status) - } - private func waitForPod(_ f: ContainerFixture, node: String, podName: String, timeoutSeconds: Int) throws { print("[k8s-net] waitForPod \(podName) on \(node) (timeout=\(timeoutSeconds)s)") - let (_, status) = try kubectl( - f, node: node, + let (_, status) = try f.kubectl( + node: node, args: [ "wait", "--for=condition=Ready", "pod/\(podName)", "--timeout=\(timeoutSeconds)s", ]) guard status == 0 else { - let (podStatus, _) = try kubectl(f, node: node, args: ["get", "pod", podName, "--no-headers"]) + let (podStatus, _) = try f.kubectl(node: node, args: ["get", "pod", podName, "--no-headers"]) print("[k8s-net] pod \(podName) status: \(podStatus.trimmingCharacters(in: .whitespacesAndNewlines))") - let (podDesc, _) = try kubectl(f, node: node, args: ["describe", "pod", podName]) + let (podDesc, _) = try f.kubectl(node: node, args: ["describe", "pod", podName]) print("[k8s-net] pod \(podName) describe:\n\(podDesc.prefix(1000))") throw CommandError.executionFailed("pod \(podName) did not become ready within \(timeoutSeconds)s") } @@ -94,8 +80,8 @@ struct TestK8sNetworkingSerial { if loadResult.status != 0 { print("[k8s-net] k8s load-image stderr: \(loadResult.error)") } #expect(loadResult.status == 0) - let (_, createStatus) = try kubectl( - f, node: name, + let (_, createStatus) = try f.kubectl( + node: name, args: [ "run", "test-pod", "--image=\(Self.testImage.rawValue)", @@ -107,8 +93,8 @@ struct TestK8sNetworkingSerial { try waitForPod(f, node: name, podName: "test-pod", timeoutSeconds: 300) - let (output, execStatus) = try kubectl( - f, node: name, + let (output, execStatus) = try f.kubectl( + node: name, args: [ "exec", "test-pod", "--", "echo", "hello", ]) @@ -143,8 +129,8 @@ struct TestK8sNetworkingSerial { #expect(loadResult.status == 0) // Server: alpine busybox httpd serving a static response. - let (_, serverStatus) = try kubectl( - f, node: name, + let (_, serverStatus) = try f.kubectl( + node: name, args: [ "run", "server", "--image=\(Self.testImage.rawValue)", @@ -158,16 +144,16 @@ struct TestK8sNetworkingSerial { try waitForPod(f, node: name, podName: "server", timeoutSeconds: 300) // Expose server as a ClusterIP service. - let (_, exposeStatus) = try kubectl( - f, node: name, + let (_, exposeStatus) = try f.kubectl( + node: name, args: [ "expose", "pod", "server", "--port=8080", "--name=server-svc", ]) #expect(exposeStatus == 0) // Client pod that stays alive so we can exec into it. - let (_, clientStatus) = try kubectl( - f, node: name, + let (_, clientStatus) = try f.kubectl( + node: name, args: [ "run", "client", "--image=\(Self.testImage.rawValue)", @@ -180,8 +166,8 @@ struct TestK8sNetworkingSerial { // Reach server via the service DNS name — exercises CoreDNS + kube-proxy. print("[k8s-net] wget from client to server-svc:8080") - let (response, wgetStatus) = try kubectl( - f, node: name, + let (response, wgetStatus) = try f.kubectl( + node: name, args: [ "exec", "client", "--", "sh", "-c", "sleep 2 && wget -qO- http://server-svc:8080", diff --git a/Tests/K8sPluginTests/K8sCreateCNITests.swift b/Tests/K8sPluginTests/K8sCreateCNITests.swift index cafde567..17e45f6b 100644 --- a/Tests/K8sPluginTests/K8sCreateCNITests.swift +++ b/Tests/K8sPluginTests/K8sCreateCNITests.swift @@ -34,6 +34,12 @@ struct K8sCreateCNIFlagTests { let command = try K8sCreate.parse(["--cni", "/tmp/my-cni.yaml"]) #expect(command.cni == "/tmp/my-cni.yaml") } + + @Test(arguments: ["NONE", "none", "None"]) + func cniAcceptsNone(value: String) throws { + let command = try K8sCreate.parse(["--cni", value]) + #expect(command.cni == value) + } } // MARK: - K8sHelper.loadCNIManifest diff --git a/docs/command-reference.md b/docs/command-reference.md index ae98bd86..d4c04329 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -1633,7 +1633,7 @@ container k8s create [--name ] [--node-image ] [--cni ] [--rm * `--name `: Cluster name (default: `k8s-dev`) * `--node-image `: Node image reference (default: `docker.io/kindest/node:v1.35.5`) -* `--cni `: Optional path to a CNI manifest to apply. If not provided, the bundled kindnet CNI is used. +* `--cni `: Optional path to a CNI manifest to apply, or `NONE` (case-insensitive) to skip installing a CNI. If not provided, the bundled kindnet CNI is used. With `NONE`, the command returns without waiting for nodes to become `Ready`, since that requires a CNI; apply your own afterward with `kubectl apply`. * `--rm`: Remove the cluster container after it stops **Resource Options** @@ -1663,6 +1663,9 @@ container k8s create --name temp-cluster --rm # create a cluster using a custom CNI manifest instead of the bundled kindnet container k8s create --cni ./my-cni.yaml + +# create a cluster with no CNI installed +container k8s create --cni NONE ``` ### `container k8s delete (rm)`