mirror of
https://github.com/apple/container.git
synced 2026-10-12 17:19:13 +00:00
192 lines
8.9 KiB
Swift
192 lines
8.9 KiB
Swift
//===----------------------------------------------------------------------===//
|
|
// Copyright © 2026 Apple Inc. and the container project authors.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// https://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
import ContainerAPIClient
|
|
import ContainerPlugin
|
|
import ContainerizationError
|
|
import ContainerizationOCI
|
|
import Foundation
|
|
import Logging
|
|
|
|
extension K8sHelper {
|
|
|
|
public static func prepareNode(nodeID: String, client: ContainerClient, log: Logger) async throws {
|
|
log.info("Preparing node", metadata: ["id": "\(nodeID)"])
|
|
let result = try await execCapture(
|
|
containerId: nodeID, executable: "/bin/sh",
|
|
arguments: ["-c", nodePrepScript], client: client)
|
|
guard result.code == 0 else {
|
|
throw ContainerizationError(.internalError, message: "node prep failed on \(nodeID): \(result.output)")
|
|
}
|
|
}
|
|
|
|
static func bootstrapControlPlane(
|
|
nodeID: String, nodeImage: String, apiServerSANs: [String], advertiseAddress: String,
|
|
schedulable: Bool, cniManifestPath: String? = nil, client: ContainerClient, log: Logger
|
|
) async throws {
|
|
let configYAML = try initConfigYAML(nodeImage: nodeImage, advertiseAddress: advertiseAddress, certSANs: apiServerSANs)
|
|
var r = try await execCapture(
|
|
containerId: nodeID, executable: "/bin/sh",
|
|
arguments: ["-c", "cat > /etc/kubernetes/kubeadm-config.yaml <<'EOF'\n\(configYAML)\nEOF"],
|
|
client: client)
|
|
guard r.code == 0 else {
|
|
throw ContainerizationError(.internalError, message: "write kubeadm config failed on \(nodeID): \(r.output)")
|
|
}
|
|
|
|
log.info("Running kubeadm init", metadata: ["node": "\(nodeID)"])
|
|
r = try await execCapture(
|
|
containerId: nodeID, executable: kubeadmPath,
|
|
arguments: [
|
|
"init", "--config", "/etc/kubernetes/kubeadm-config.yaml",
|
|
"--ignore-preflight-errors", ignorePreflightErrors,
|
|
],
|
|
client: client)
|
|
guard r.code == 0 else {
|
|
throw ContainerizationError(.internalError, message: "kubeadm init failed on \(nodeID): \(r.output)")
|
|
}
|
|
|
|
r = try await execCapture(
|
|
containerId: nodeID, executable: "/bin/sh",
|
|
arguments: ["-c", "mkdir -p /root/.kube && cp \(kubeconfigPath) /root/.kube/config"],
|
|
client: client)
|
|
guard r.code == 0 else {
|
|
throw ContainerizationError(.internalError, message: "failed to install root kubeconfig on \(nodeID): \(r.output)")
|
|
}
|
|
|
|
if schedulable {
|
|
log.info("Removing control-plane taint for single-node scheduling", metadata: ["node": "\(nodeID)"])
|
|
let taintCode = try await runProbe(
|
|
client: client, containerId: nodeID,
|
|
arguments: ["taint", "nodes", "--all", "node-role.kubernetes.io/control-plane-"])
|
|
guard taintCode == 0 else {
|
|
throw ContainerizationError(.internalError, message: "failed to remove control-plane taint on \(nodeID)")
|
|
}
|
|
}
|
|
|
|
if cniManifestPath?.uppercased() == noCNIName {
|
|
log.info("Skipping CNI installation", metadata: ["node": "\(nodeID)"])
|
|
} else {
|
|
log.info("Applying CNI manifest", metadata: ["node": "\(nodeID)"])
|
|
let manifest = try await loadCNIManifest(path: cniManifestPath, log: log)
|
|
let apply = "\(kubeconfigEnv) kubectl apply -f - <<'EOF'\n\(manifest)\nEOF"
|
|
r = try await execCapture(
|
|
containerId: nodeID, executable: "/bin/sh",
|
|
arguments: ["-c", apply], client: client)
|
|
guard r.code == 0 else {
|
|
throw ContainerizationError(.internalError, message: "apply CNI failed on \(nodeID): \(r.output)")
|
|
}
|
|
}
|
|
}
|
|
|
|
static func createJoinToken(nodeID: String, client: ContainerClient) async throws -> (token: String, caCertHash: String) {
|
|
let (code, output) = try await execCapture(
|
|
containerId: nodeID, executable: kubeadmPath,
|
|
arguments: ["token", "create", "--print-join-command"],
|
|
client: client)
|
|
guard code == 0 else {
|
|
throw ContainerizationError(.internalError, message: "kubeadm token create failed on \(nodeID): \(output)")
|
|
}
|
|
let parts = output.trimmingCharacters(in: .whitespacesAndNewlines).components(separatedBy: " ")
|
|
guard let tokenIdx = parts.firstIndex(of: "--token"), tokenIdx + 1 < parts.count,
|
|
let hashIdx = parts.firstIndex(of: "--discovery-token-ca-cert-hash"), hashIdx + 1 < parts.count
|
|
else {
|
|
throw ContainerizationError(.internalError, message: "could not parse join command output from kubeadm on \(nodeID)")
|
|
}
|
|
return (token: parts[tokenIdx + 1], caCertHash: parts[hashIdx + 1])
|
|
}
|
|
|
|
static func loadCNIManifest(path: String?, log: Logger) async throws -> String {
|
|
if let path {
|
|
do {
|
|
return try String(contentsOfFile: path, encoding: .utf8)
|
|
} catch {
|
|
throw ContainerizationError(.invalidArgument, message: "failed to read CNI manifest at \(path): \(error)")
|
|
}
|
|
}
|
|
return try await loadKindnetManifest(log: log)
|
|
}
|
|
|
|
private static func loadKindnetManifest(log: Logger) async throws -> String {
|
|
let pluginLoader = try await Utility.createPluginLoader(log: log)
|
|
guard let plugin = pluginLoader.findPlugin(forExecutable: CommandLine.executablePath),
|
|
let resourceURL = plugin.resourceURL
|
|
else {
|
|
throw ContainerizationError(.internalError, message: "unable to locate k8s plugin installation or resources")
|
|
}
|
|
let url = resourceURL.appendingPathComponent("kindnet.yaml")
|
|
guard let contents = try? String(contentsOf: url, encoding: .utf8) else {
|
|
throw ContainerizationError(.internalError, message: "kindnet manifest resource missing at \(url.path)")
|
|
}
|
|
return contents
|
|
}
|
|
|
|
private static var nodePrepScript: String {
|
|
"""
|
|
set -e
|
|
mkdir -p /etc/containerd/conf.d
|
|
cat > /etc/containerd/conf.d/native-snapshotter.toml <<'EOF'
|
|
[plugins.'io.containerd.cri.v1.images']
|
|
snapshotter = "native"
|
|
EOF
|
|
sysctl -w net.ipv4.ip_forward=1 2>/dev/null || true
|
|
sysctl -w net.bridge.bridge-nf-call-iptables=1 2>/dev/null || true
|
|
sysctl -w net.bridge.bridge-nf-call-ip6tables=1 2>/dev/null || true
|
|
systemctl restart containerd
|
|
ctr -n k8s.io images tag registry.k8s.io/pause:3.10 registry.k8s.io/pause:3.10.1 2>/dev/null || true
|
|
/usr/sbin/iptables -t mangle -A OUTPUT -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1220
|
|
/usr/sbin/iptables -t mangle -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1220
|
|
"""
|
|
}
|
|
|
|
private static func initConfigYAML(nodeImage: String, advertiseAddress: String, certSANs: [String]) throws -> String {
|
|
let sans = certSANs.map { " - \($0)" }.joined(separator: "\n")
|
|
return """
|
|
apiVersion: kubeadm.k8s.io/v1beta4
|
|
kind: InitConfiguration
|
|
localAPIEndpoint:
|
|
advertiseAddress: \(advertiseAddress)
|
|
bindPort: 6443
|
|
nodeRegistration:
|
|
criSocket: unix:///run/containerd/containerd.sock
|
|
---
|
|
apiVersion: kubeadm.k8s.io/v1beta4
|
|
kind: ClusterConfiguration
|
|
kubernetesVersion: \(try kubernetesVersion(nodeImage: nodeImage))
|
|
networking:
|
|
podSubnet: \(podSubnet)
|
|
apiServer:
|
|
certSANs:
|
|
\(sans)
|
|
---
|
|
apiVersion: kubelet.config.k8s.io/v1beta1
|
|
kind: KubeletConfiguration
|
|
cgroupDriver: systemd
|
|
failSwapOn: false
|
|
"""
|
|
}
|
|
|
|
/// kubeadm needs the exact version, and only the tag carries it.
|
|
static func kubernetesVersion(nodeImage: String) throws -> String {
|
|
let nameAndTag = nodeImage.split(separator: "@").first.map(String.init) ?? nodeImage
|
|
guard let ref = try? Reference.parse(nameAndTag), let tag = ref.tag else {
|
|
throw ContainerizationError(
|
|
.invalidArgument,
|
|
message: "node image \(nodeImage) has no tag; use a tagged image such as docker.io/kindest/node:v1.34.11")
|
|
}
|
|
return tag
|
|
}
|
|
}
|