diff --git a/Package.swift b/Package.swift index a07e366d..99e44b2a 100644 --- a/Package.swift +++ b/Package.swift @@ -39,11 +39,19 @@ let package = Package( .library(name: "ContainerPlugin", targets: ["ContainerPlugin"]), .library(name: "ContainerXPC", targets: ["ContainerXPC"]), .library(name: "SocketForwarder", targets: ["SocketForwarder"]), + .library(name: "ContainerBuildReporting", targets: ["ContainerBuildReporting"]), + .library(name: "ContainerBuildIR", targets: ["ContainerBuildIR"]), + .library(name: "ContainerBuildExecutor", targets: ["ContainerBuildExecutor"]), + .library(name: "ContainerBuildCache", targets: ["ContainerBuildCache"]), + .library(name: "ContainerBuildSnapshotter", targets: ["ContainerBuildSnapshotter"]), + .library(name: "ContainerBuildDiffer", targets: ["ContainerBuildDiffer"]), + .library(name: "ContainerBuildParser", targets: ["ContainerBuildParser"]), ], dependencies: [ .package(url: "https://github.com/apple/swift-log.git", from: "1.0.0"), .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.3.0"), .package(url: "https://github.com/apple/swift-collections.git", from: "1.2.0"), + .package(url: "https://github.com/apple/swift-crypto.git", from: "3.0.0"), .package(url: "https://github.com/grpc/grpc-swift.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.29.0"), .package(url: "https://github.com/apple/swift-nio.git", from: "2.80.0"), @@ -225,6 +233,98 @@ let package = Package( "ContainerClient", ] ), + .executableTarget( + name: "native-builder-demo", + dependencies: ["ContainerBuildIR", "ContainerBuildExecutor", "ContainerBuildCache"], + path: "Sources/NativeBuilder/ContainerBuildDemo" + ), + .target( + name: "ContainerBuildReporting", + dependencies: [], + path: "Sources/NativeBuilder/ContainerBuildReporting", + swiftSettings: [ + .enableExperimentalFeature("StrictConcurrency") + ] + ), + .target( + name: "ContainerBuildIR", + dependencies: [ + "ContainerBuildReporting", + .product(name: "ContainerizationOCI", package: "containerization"), + .product(name: "Crypto", package: "swift-crypto"), + ], + path: "Sources/NativeBuilder/ContainerBuildIR", + swiftSettings: [ + .enableExperimentalFeature("StrictConcurrency") + ], + ), + .target( + name: "ContainerBuildExecutor", + dependencies: [ + "ContainerBuildReporting", + "ContainerBuildIR", + "ContainerBuildSnapshotter", + "ContainerBuildCache", + .product(name: "ContainerizationOCI", package: "containerization"), + ], + path: "Sources/NativeBuilder/ContainerBuildExecutor", + swiftSettings: [ + .enableExperimentalFeature("StrictConcurrency") + ] + ), + .target( + name: "ContainerBuildCache", + dependencies: [ + "ContainerBuildIR", + "ContainerBuildSnapshotter", + .product(name: "ContainerizationOCI", package: "containerization"), + .product(name: "ContainerizationExtras", package: "containerization"), + .product(name: "Crypto", package: "swift-crypto"), + ], + path: "Sources/NativeBuilder/ContainerBuildCache", + swiftSettings: [ + .enableExperimentalFeature("StrictConcurrency") + ] + ), + .target( + name: "ContainerBuildSnapshotter", + dependencies: ["ContainerBuildIR"], + path: "Sources/NativeBuilder/ContainerBuildSnapshotter", + swiftSettings: [ + .enableExperimentalFeature("StrictConcurrency") + ] + ), + .target( + name: "ContainerBuildDiffer", + dependencies: [ + "ContainerBuildIR", + "ContainerBuildSnapshotter", + ], + path: "Sources/NativeBuilder/ContainerBuildDiffer", + swiftSettings: [ + .enableExperimentalFeature("StrictConcurrency") + ] + ), + .target( + name: "ContainerBuildParser", + dependencies: [ + "ContainerBuildIR" + ], + path: "Sources/NativeBuilder/ContainerBuildParser", + swiftSettings: [ + .enableExperimentalFeature("StrictConcurrency") + ] + ), + .testTarget( + name: "NativeBuilderTests", + dependencies: [ + "ContainerBuildIR", + "ContainerBuildExecutor", + "ContainerBuildCache", + "ContainerBuildReporting", + "ContainerBuildParser", + ] + ), .target( name: "ContainerPersistence", dependencies: [ diff --git a/Sources/NativeBuilder/ContainerBuildCache/BuildCache.swift b/Sources/NativeBuilder/ContainerBuildCache/BuildCache.swift new file mode 100644 index 00000000..0dff1256 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildCache/BuildCache.swift @@ -0,0 +1,188 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildSnapshotter +import ContainerizationOCI +import Foundation + +/// Protocol for build cache implementations. +/// +/// The cache stores operation results to avoid redundant execution. +/// +/// PERFORMANCE OPTIMIZATION OPPORTUNITIES (from Design.md): +/// +/// 1. Graph-Level Cache Analysis: +/// - Add a method to check multiple cache keys in parallel before execution starts +/// - Return a map of cache hits/misses for the entire build graph +/// - Example: func checkBatch(_ keys: [CacheKey]) async -> [CacheKey: Bool] +/// +/// 2. Cache-Aware Graph Transformation: +/// - Implement a CachedScheduler that pre-processes the build graph +/// - Replace cached operations with lightweight cache retrieval operations +/// - Skip entire dependency chains if final results are cached +/// +/// 3. Speculative Cache Warming: +/// - Add methods to prefetch likely cache entries based on build patterns +/// - Support background cache population for common operations +/// +/// 4. Parallel Cache Operations: +/// - Batch get/put operations for multiple artifacts +/// - Use Swift concurrency for non-blocking cache access +/// - Example: func getBatch(_ keys: [CacheKey]) async -> [CacheKey: CachedResult?] +/// +/// 5. Cache Metadata for Scheduling: +/// - Expose cache hit probability estimates +/// - Provide operation cost estimates based on historical data +/// - Enable scheduler to make smarter execution decisions +/// +/// Current implementation is functionally correct but checks cache sequentially +/// during execution rather than optimizing the execution graph based on cache state. +public protocol BuildCache: Sendable { + /// Look up a cached result for an operation. + /// + /// - Parameters: + /// - key: The cache key for the operation + /// - operation: The operation being cached + /// - Returns: The cached result if found, nil otherwise + func get(_ key: CacheKey, for operation: ContainerBuildIR.Operation) async -> CachedResult? + + /// Store a result in the cache. + /// + /// - Parameters: + /// - result: The result to cache + /// - key: The cache key + /// - operation: The operation that produced the result + func put(_ result: CachedResult, key: CacheKey, for operation: ContainerBuildIR.Operation) async + + /// Get cache statistics. + /// + /// - Returns: Statistics about cache usage and performance + func statistics() async -> CacheStatistics +} + +/// A key for cache lookups. +public struct CacheKey: Hashable, Sendable { + /// The digest of the operation. + public let operationDigest: ContainerBuildIR.Digest + + /// Input digests from dependencies. + public let inputDigests: [ContainerBuildIR.Digest] + + /// Platform identifier. + public let platform: Platform + + public init( + operationDigest: ContainerBuildIR.Digest, + inputDigests: [ContainerBuildIR.Digest] = [], + platform: Platform + ) { + self.operationDigest = operationDigest + self.inputDigests = inputDigests + self.platform = platform + } +} + +/// A cached execution result. +public struct CachedResult: Sendable { + /// The snapshot produced by the operation. + public let snapshot: Snapshot + + /// Environment changes made by the operation. + public let environmentChanges: [String: EnvironmentValue] + + /// Metadata changes. + public let metadataChanges: [String: String] + + public init( + snapshot: Snapshot, + environmentChanges: [String: EnvironmentValue] = [:], + metadataChanges: [String: String] = [:] + ) { + self.snapshot = snapshot + self.environmentChanges = environmentChanges + self.metadataChanges = metadataChanges + } +} + +/// A memory-based cache implementation for development/testing. +public actor MemoryBuildCache: BuildCache { + private var storage: [CacheKey: CachedResult] = [:] + private var hits: Int = 0 + private var misses: Int = 0 + + public init() {} + + public func get(_ key: CacheKey, for operation: ContainerBuildIR.Operation) async -> CachedResult? { + guard let result = storage[key] else { + misses += 1 + return nil + } + hits += 1 + return result + } + + public func put(_ result: CachedResult, key: CacheKey, for operation: ContainerBuildIR.Operation) async { + storage[key] = result + } + + public func statistics() async -> CacheStatistics { + CacheStatistics( + entryCount: storage.count, + totalSize: UInt64(storage.count * 1024), // Rough estimate + hitRate: hits + misses > 0 ? Double(hits) / Double(hits + misses) : 0, + oldestEntryAge: 0, + mostRecentEntryAge: 0, + evictionPolicy: "none", + compressionRatio: 1.0, + averageEntrySize: 1024, + operationMetrics: .empty, + errorCount: 0, + lastGCTime: nil, + shardInfo: nil + ) + } +} + +/// A no-op cache implementation that never caches. +public struct NoOpBuildCache: BuildCache { + public init() {} + + public func get(_ key: CacheKey, for operation: ContainerBuildIR.Operation) async -> CachedResult? { + nil + } + + public func put(_ result: CachedResult, key: CacheKey, for operation: ContainerBuildIR.Operation) async { + // No-op + } + + public func statistics() async -> CacheStatistics { + CacheStatistics( + entryCount: 0, + totalSize: 0, + hitRate: 0, + oldestEntryAge: 0, + mostRecentEntryAge: 0, + evictionPolicy: "none", + compressionRatio: 1.0, + averageEntrySize: 0, + operationMetrics: .empty, + errorCount: 0, + lastGCTime: nil, + shardInfo: nil + ) + } +} diff --git a/Sources/NativeBuilder/ContainerBuildCache/CacheError.swift b/Sources/NativeBuilder/ContainerBuildCache/CacheError.swift new file mode 100644 index 00000000..f04987b0 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildCache/CacheError.swift @@ -0,0 +1,53 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import Foundation + +/// An error related to cache operations. +public enum CacheError: Error, LocalizedError { + /// A cache entry was expected but not found. This is often a normal cache miss. + case itemNotFound(key: String) + + /// The manifest file for the cache is corrupted or unreadable. + case manifestUnreadable(path: String, underlyingError: Error) + + /// A file was read from the cache, but its content hash did not match the expected digest. + case digestMismatch(expected: Digest, actual: Digest) + + /// An error occurred while trying to write an item to the cache storage. + case storageFailed(path: String, underlyingError: Error) + + /// Failed to encode cache-related data as UTF-8. + case encodingFailed(String) + + // MARK: - LocalizedError Conformance + + public var errorDescription: String? { + switch self { + case .itemNotFound(let key): + return "Item with key '\(key)' not found in cache." + case .manifestUnreadable(let path, _): + return "Failed to read cache manifest at '\(path)'." + case .digestMismatch(let expected, let actual): + return "Cache integrity check failed: content digest \(actual) does not match expected digest \(expected)." + case .storageFailed(let path, _): + return "Failed to write to cache storage at '\(path)'." + case .encodingFailed(let details): + return "Failed to encode cache data: \(details)" + } + } +} diff --git a/Sources/NativeBuilder/ContainerBuildCache/CacheIndex.swift b/Sources/NativeBuilder/ContainerBuildCache/CacheIndex.swift new file mode 100644 index 00000000..ceeeff04 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildCache/CacheIndex.swift @@ -0,0 +1,290 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerizationExtras +import ContainerizationOCI +import Foundation + +// MARK: - Cache Index + +/// Actor-based cache index that manages cache metadata with atomic updates +public actor CacheIndex: Sendable { + private let path: URL + + /// Cache index state persisted to cache.json + struct State: Codable { + var entries: [String: CacheEntry] + var version: Int + var statistics: Statistics + + struct Statistics: Codable { + var totalSize: UInt64 + var entryCount: Int + var hitCount: Int64 + var missCount: Int64 + var evictionCount: Int64 + var lastModified: Date + var lastGC: Date? + } + + static let empty = State( + entries: [:], + version: 1, + statistics: Statistics( + totalSize: 0, + entryCount: 0, + hitCount: 0, + missCount: 0, + evictionCount: 0, + lastModified: Date(), + lastGC: nil + ) + ) + } + + public init(path: URL) throws { + try FileManager.default.createDirectory(at: path, withIntermediateDirectories: true) + self.path = path + } + + // MARK: - Public Methods + + /// Add or update a cache entry + public func put( + key: String, + descriptor: Descriptor, + metadata: CacheMetadata + ) async throws { + var state = try self.load() + + // Update or create entry + let entry = CacheEntry( + descriptor: descriptor, + metadata: metadata + ) + + // Update statistics + if state.entries[key] == nil { + state.statistics.entryCount += 1 + } + state.entries[key] = entry + state.statistics.totalSize = calculateTotalSize(state.entries) + state.statistics.lastModified = Date() + + try self.save(state) + } + + /// Get a cache entry and update access time + public func get(key: String) async throws -> CacheEntry? { + var state = try self.load() + + guard var entry = state.entries[key] else { + state.statistics.missCount += 1 + try self.save(state) + return nil + } + + // Update access time + entry.metadata.accessedAt = Date() + state.entries[key] = entry + state.statistics.hitCount += 1 + state.statistics.lastModified = Date() + + try self.save(state) + return entry + } + + /// Remove cache entries + public func remove(keys: [String]) async throws { + var state = try self.load() + + for key in keys { + if state.entries.removeValue(forKey: key) != nil { + state.statistics.entryCount -= 1 + state.statistics.evictionCount += 1 + } + } + + state.statistics.totalSize = calculateTotalSize(state.entries) + state.statistics.lastModified = Date() + + try self.save(state) + } + + /// Get all cache entries + public func allEntries() async throws -> [String: CacheEntry] { + let state = try self.load() + return state.entries + } + + /// Get cache statistics + public func statistics() async throws -> CacheStatistics { + let state = try self.load() + + // Calculate derived statistics + let hitRate = + state.statistics.hitCount + state.statistics.missCount > 0 + ? Double(state.statistics.hitCount) / Double(state.statistics.hitCount + state.statistics.missCount) + : 0.0 + + let ages = state.entries.values.map { entry in + Date().timeIntervalSince(entry.metadata.createdAt) + }.sorted() + + let oldestAge = ages.last ?? 0 + let newestAge = ages.first ?? 0 + + let avgSize = + state.statistics.entryCount > 0 + ? state.statistics.totalSize / UInt64(state.statistics.entryCount) + : 0 + + return CacheStatistics( + entryCount: state.statistics.entryCount, + totalSize: state.statistics.totalSize, + hitRate: hitRate, + oldestEntryAge: oldestAge, + mostRecentEntryAge: newestAge, + evictionPolicy: "lru", + compressionRatio: 1.0, // TODO: Calculate actual compression ratio + averageEntrySize: avgSize, + operationMetrics: .empty, + errorCount: 0, + lastGCTime: state.statistics.lastGC, + shardInfo: nil + ) + } + + // MARK: - File Operations + + /// Load cache index from cache.json + private func load() throws -> State { + let indexPath = self.path.appendingPathComponent("cache.json") + + guard FileManager.default.fileExists(atPath: indexPath.path) else { + return .empty + } + + do { + let data = try Data(contentsOf: indexPath) + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + return try decoder.decode(State.self, from: data) + } catch { + // Handle corrupted index by starting fresh + print("Warning: Cache index corrupted, starting with empty cache: \(error.localizedDescription)") + + // Try to backup the corrupted file for debugging + let backupPath = indexPath.appendingPathExtension("corrupted") + try? FileManager.default.moveItem(at: indexPath, to: backupPath) + + // Return empty state to start fresh + return .empty + } + } + + /// Save cache index to cache.json atomically + private func save(_ state: State) throws { + let indexPath = self.path.appendingPathComponent("cache.json") + let tempPath = indexPath.appendingPathExtension("tmp") + + do { + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .iso8601 + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + + let data = try encoder.encode(state) + try data.write(to: tempPath, options: .atomic) + + // Atomic rename + _ = try FileManager.default.replaceItem(at: indexPath, withItemAt: tempPath, backupItemName: nil, options: [], resultingItemURL: nil) + } catch { + // Clean up temp file if it exists + try? FileManager.default.removeItem(at: tempPath) + throw CacheError.storageFailed(path: tempPath.path, underlyingError: error) + } + } + + // MARK: - Helper Methods + + private func calculateTotalSize(_ entries: [String: CacheEntry]) -> UInt64 { + entries.values.reduce(0) { total, entry in + total + UInt64(entry.descriptor.size) + } + } +} + +// MARK: - Cache Entry + +/// Individual cache entry containing manifest descriptor and metadata +public struct CacheEntry: Codable, Sendable { + /// OCI descriptor for the cache manifest + public let descriptor: Descriptor + + /// Cache metadata + public var metadata: CacheMetadata + + public init(descriptor: Descriptor, metadata: CacheMetadata) { + self.descriptor = descriptor + self.metadata = metadata + } +} + +// MARK: - Cache Metadata + +/// Metadata associated with a cache entry +public struct CacheMetadata: Codable, Sendable { + /// When the entry was created + public let createdAt: Date + + /// When the entry was last accessed + public var accessedAt: Date + + /// Hash of the operation that created this cache entry + public let operationHash: String + + /// Platform this cache entry is for + public let platform: Platform + + /// Time-to-live in seconds (nil means no expiration) + public let ttl: TimeInterval? + + /// Custom tags for filtering + public let tags: [String: String] + + public init( + createdAt: Date = Date(), + accessedAt: Date = Date(), + operationHash: String, + platform: Platform, + ttl: TimeInterval? = nil, + tags: [String: String] = [:] + ) { + self.createdAt = createdAt + self.accessedAt = accessedAt + self.operationHash = operationHash + self.platform = platform + self.ttl = ttl + self.tags = tags + } + + /// Check if the entry has expired + public var isExpired: Bool { + guard let ttl = ttl else { return false } + return Date().timeIntervalSince(createdAt) > ttl + } +} diff --git a/Sources/NativeBuilder/ContainerBuildCache/CacheManifest.swift b/Sources/NativeBuilder/ContainerBuildCache/CacheManifest.swift new file mode 100644 index 00000000..81f4ac84 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildCache/CacheManifest.swift @@ -0,0 +1,189 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerizationOCI +import Foundation + +// MARK: - Cache Manifest Types + +/// OCI-compliant cache manifest stored in ContentStore +struct CacheManifest: Codable, Sendable { + let schemaVersion: Int + let mediaType: String + let config: CacheConfig + let layers: [CacheLayer] + let annotations: [String: String] + let subject: Descriptor? + + static let currentSchemaVersion = 2 + static let manifestMediaType = "application/vnd.container-build.cache.manifest.v2+json" + + init( + schemaVersion: Int = CacheManifest.currentSchemaVersion, + mediaType: String = CacheManifest.manifestMediaType, + config: CacheConfig, + layers: [CacheLayer], + annotations: [String: String] = [:], + subject: Descriptor? = nil + ) { + self.schemaVersion = schemaVersion + self.mediaType = mediaType + self.config = config + self.layers = layers + self.annotations = annotations + self.subject = subject + } + + func allContentDigests() -> [String] { + layers.map { $0.descriptor.digest } + } +} + +/// Cache configuration embedded in manifest +struct CacheConfig: Codable, Sendable { + let cacheKey: SerializedCacheKey + let operationType: String + let platform: Platform + let buildVersion: String + let createdAt: Date + + init( + cacheKey: SerializedCacheKey, + operationType: String, + platform: Platform, + buildVersion: String, + createdAt: Date = Date() + ) { + self.cacheKey = cacheKey + self.operationType = operationType + self.platform = platform + self.buildVersion = buildVersion + self.createdAt = createdAt + } +} + +/// Cache layer representing a component of the cached result +struct CacheLayer: Codable, Sendable { + let descriptor: Descriptor + let type: LayerType + + enum LayerType: String, Codable, Sendable { + case snapshot = "snapshot" + case environment = "environment" + case metadata = "metadata" + } +} + +/// Serializable version of CacheKey for storage +struct SerializedCacheKey: Codable, Sendable { + let operationDigest: String + let inputDigests: [String] + let platform: PlatformData + + struct PlatformData: Codable, Sendable { + let os: String + let architecture: String + let variant: String? + let osVersion: String? + let osFeatures: [String]? + } + + init(from key: CacheKey) { + self.operationDigest = key.operationDigest.stringValue + self.inputDigests = key.inputDigests.map { $0.stringValue } + self.platform = PlatformData( + os: key.platform.os, + architecture: key.platform.architecture, + variant: key.platform.variant, + osVersion: key.platform.osVersion, + osFeatures: key.platform.osFeatures.map { Array($0) } + ) + } +} + +// MARK: - Manifest Extensions + +extension CacheManifest { + /// Create a manifest with subject reference (for linking to base images) + func withSubject(_ subject: Descriptor) -> CacheManifest { + CacheManifest( + schemaVersion: schemaVersion, + mediaType: mediaType, + config: config, + layers: layers, + annotations: annotations, + subject: subject + ) + } + + /// Add or update annotation + func withAnnotation(key: String, value: String) -> CacheManifest { + var newAnnotations = annotations + newAnnotations[key] = value + + return CacheManifest( + schemaVersion: schemaVersion, + mediaType: mediaType, + config: config, + layers: layers, + annotations: newAnnotations, + subject: subject + ) + } + + /// Get total size of all layers + var totalSize: Int64 { + layers.reduce(0) { $0 + $1.descriptor.size } + } + + /// Check if manifest is compressed + var isCompressed: Bool { + layers.contains { layer in + layer.descriptor.mediaType.contains("gzip") || layer.descriptor.mediaType.contains("zstd") || layer.descriptor.mediaType.contains("lz4") + } + } +} + +// MARK: - Descriptor Extensions + +extension Descriptor { + /// Create a descriptor for cache content + static func forCacheContent( + mediaType: String, + digest: String, + size: Int64, + compressed: Bool = false, + annotations: [String: String]? = nil + ) -> Descriptor { + var finalMediaType = mediaType + if compressed && !mediaType.contains("+") { + // Detect compression from annotations if not in media type + if let compressionType = annotations?["com.apple.container-build.compression"] { + finalMediaType += "+\(compressionType)" + } + } + + return Descriptor( + mediaType: finalMediaType, + digest: digest, + size: size, + urls: nil, + annotations: annotations, + platform: nil + ) + } +} diff --git a/Sources/NativeBuilder/ContainerBuildCache/CacheTypes.swift b/Sources/NativeBuilder/ContainerBuildCache/CacheTypes.swift new file mode 100644 index 00000000..05c4d647 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildCache/CacheTypes.swift @@ -0,0 +1,284 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerizationOCI +import Foundation + +// MARK: - Configuration Types + +/// Cache configuration +public struct CacheConfiguration: Sendable { + /// Maximum cache size in bytes + public let maxSize: UInt64 + + /// Maximum age for cache entries + public let maxAge: TimeInterval + + /// Compression configuration + public let compression: CompressionConfiguration + + /// Index database path + public let indexPath: URL + + /// Eviction policy + public let evictionPolicy: EvictionPolicy + + /// Concurrency limits + public let concurrency: ConcurrencyConfiguration + + /// Integrity verification + public let verifyIntegrity: Bool + + /// Shard configuration for distributed caching + public let sharding: ShardingConfiguration? + + /// Garbage collection interval + public let gcInterval: TimeInterval + + /// Cache key version for invalidation + public let cacheKeyVersion: String + + /// Default TTL for cache entries + public let defaultTTL: TimeInterval? + + public init( + maxSize: UInt64 = 10 * 1024 * 1024 * 1024, // 10GB default + maxAge: TimeInterval = 7 * 24 * 60 * 60, // 7 days default + compression: CompressionConfiguration = .default, + indexPath: URL = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first! + .appendingPathComponent("com.apple.container-build.cache.db"), + evictionPolicy: EvictionPolicy = .lru, + concurrency: ConcurrencyConfiguration = .default, + verifyIntegrity: Bool = true, + sharding: ShardingConfiguration? = nil, + gcInterval: TimeInterval = 3600, // 1 hour + cacheKeyVersion: String = "v1", + defaultTTL: TimeInterval? = nil + ) { + self.maxSize = maxSize + self.maxAge = maxAge + self.compression = compression + self.indexPath = indexPath + self.evictionPolicy = evictionPolicy + self.concurrency = concurrency + self.verifyIntegrity = verifyIntegrity + self.sharding = sharding + self.gcInterval = gcInterval + self.cacheKeyVersion = cacheKeyVersion + self.defaultTTL = defaultTTL + } +} + +public struct CompressionConfiguration: Sendable { + public let algorithm: CompressionAlgorithm + public let level: Int + public let minSize: Int // Minimum size to compress + + public enum CompressionAlgorithm: String, Sendable { + case zstd = "zstd" + case lz4 = "lz4" + case gzip = "gzip" + case none = "none" + } + + public static let `default` = CompressionConfiguration( + algorithm: .zstd, + level: 3, + minSize: 1024 // 1KB + ) + + public init(algorithm: CompressionAlgorithm, level: Int, minSize: Int) { + self.algorithm = algorithm + self.level = level + self.minSize = minSize + } +} + +public enum EvictionPolicy: String, Sendable { + case lru = "lru" // Least Recently Used + case lfu = "lfu" // Least Frequently Used + case fifo = "fifo" // First In First Out + case ttl = "ttl" // Time To Live based + case arc = "arc" // Adaptive Replacement Cache +} + +public struct ConcurrencyConfiguration: Sendable { + public let maxConcurrentReads: Int + public let maxConcurrentWrites: Int + public let maxConcurrentEvictions: Int + + public static let `default` = ConcurrencyConfiguration( + maxConcurrentReads: 100, + maxConcurrentWrites: 10, + maxConcurrentEvictions: 2 + ) + + public init(maxConcurrentReads: Int, maxConcurrentWrites: Int, maxConcurrentEvictions: Int) { + self.maxConcurrentReads = maxConcurrentReads + self.maxConcurrentWrites = maxConcurrentWrites + self.maxConcurrentEvictions = maxConcurrentEvictions + } +} + +public struct ShardingConfiguration: Sendable { + public let shardCount: Int + public let shardId: Int + public let consistentHashing: Bool + + public init(shardCount: Int, shardId: Int, consistentHashing: Bool = true) { + self.shardCount = shardCount + self.shardId = shardId + self.consistentHashing = consistentHashing + } +} + +// MARK: - Cache Entry Types + +/// Cache index entry for tracking +struct CacheIndexEntry: Sendable, Codable { + let digest: String + let manifestSize: Int64 + let totalSize: UInt64 + let createdAt: Date + var lastAccessedAt: Date + var accessCount: Int64 + let platform: Platform + let operationType: String + let contentDigests: [String] + let compression: String + let transaction: UUID? + + var age: TimeInterval { + Date().timeIntervalSince(createdAt) + } +} + +/// Normalized platform for consistent hashing +struct NormalizedPlatform: Codable { + let os: String + let architecture: String + let variant: String? + let osVersion: String? + let osFeatures: [String]? +} + +// MARK: - Statistics Types + +/// Enhanced cache statistics +public struct CacheStatistics: Sendable { + public let entryCount: Int + public let totalSize: UInt64 + public let hitRate: Double + public let oldestEntryAge: TimeInterval + public let mostRecentEntryAge: TimeInterval + public let evictionPolicy: String + public let compressionRatio: Double + public let averageEntrySize: UInt64 + public let operationMetrics: OperationMetrics + public let errorCount: Int + public let lastGCTime: Date? + public let shardInfo: ShardInfo? + + public static let empty = CacheStatistics( + entryCount: 0, + totalSize: 0, + hitRate: 0, + oldestEntryAge: 0, + mostRecentEntryAge: 0, + evictionPolicy: "none", + compressionRatio: 1.0, + averageEntrySize: 0, + operationMetrics: .empty, + errorCount: 0, + lastGCTime: nil, + shardInfo: nil + ) +} + +public struct OperationMetrics: Sendable { + public let totalOperations: Int64 + public let averageGetDuration: TimeInterval + public let averagePutDuration: TimeInterval + public let p95GetDuration: TimeInterval + public let p95PutDuration: TimeInterval + + public static let empty = OperationMetrics( + totalOperations: 0, + averageGetDuration: 0, + averagePutDuration: 0, + p95GetDuration: 0, + p95PutDuration: 0 + ) +} + +public struct ShardInfo: Sendable { + public let shardId: Int + public let totalShards: Int +} + +// MARK: - Operation Types + +/// Clear filter for selective cache clearing +public struct ClearFilter: Sendable { + public let platform: Platform? + public let operationType: String? + public let olderThan: Date? + public let pattern: String? + + public init( + platform: Platform? = nil, + operationType: String? = nil, + olderThan: Date? = nil, + pattern: String? = nil + ) { + self.platform = platform + self.operationType = operationType + self.olderThan = olderThan + self.pattern = pattern + } +} + +// MARK: - Eviction Types + +enum EvictionReason: String { + case sizeLimit = "size_limit" + case expired = "expired" + case manual = "manual" + case lowMemory = "low_memory" +} + +// MARK: - Extensions + +extension Array { + func chunked(into size: Int) -> [[Element]] { + stride(from: 0, to: count, by: size).map { + Array(self[$0.. String { + map { String(format: "%02x", $0) }.joined() + } +} + +extension ContainerBuildIR.Digest { + func hexEncodedString() -> String { + self.bytes.map { String(format: "%02x", $0) }.joined() + } +} diff --git a/Sources/NativeBuilder/ContainerBuildCache/ContentAddressableCache.swift b/Sources/NativeBuilder/ContainerBuildCache/ContentAddressableCache.swift new file mode 100644 index 00000000..2fa69693 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildCache/ContentAddressableCache.swift @@ -0,0 +1,526 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildSnapshotter +import ContainerizationOCI +import Crypto +import Foundation + +/// Content-addressable cache implementation using ContentStore as backing storage. +/// +/// This implementation stores cache entries as OCI artifacts with manifests +/// pointing to content layers. It provides atomic operations, deduplication, +/// and efficient eviction policies. +public actor ContentAddressableCache: BuildCache { + private let contentStore: ContentStore + private let index: CacheIndex + private let configuration: CacheConfiguration + private var evictionTask: Task? + + /// Initialize a new content-addressable cache. + /// + /// - Parameters: + /// - contentStore: The content store for storage + /// - configuration: Cache configuration + public init( + contentStore: ContentStore, + configuration: CacheConfiguration = CacheConfiguration() + ) async throws { + self.contentStore = contentStore + self.configuration = configuration + do { + self.index = try CacheIndex(path: configuration.indexPath) + } catch { + throw CacheError.manifestUnreadable(path: configuration.indexPath.path, underlyingError: error) + } + + // Start background eviction task + self.evictionTask = Task { [weak self] in + await self?.runPeriodicEviction() + } + } + + deinit { + evictionTask?.cancel() + } + + // MARK: - BuildCache Protocol Implementation + + public func get(_ key: CacheKey, for operation: ContainerBuildIR.Operation) async -> CachedResult? { + // Generate cache digest + guard let digest = try? generateCacheDigest(from: key) else { + return nil + } + + // Check index + guard let entry = try? await index.get(key: digest) else { + return nil + } + + // Fetch manifest from content store using the manifest digest from the index + guard let manifest: CacheManifest = try? await contentStore.get(digest: entry.descriptor.digest) else { + // Clean up orphaned index entry + try? await index.remove(keys: [digest]) + return nil + } + + // Reconstruct result from manifest + guard let result = try? await reconstructResult(from: manifest) else { + return nil + } + + // Update access time + var updatedMetadata = entry.metadata + updatedMetadata.accessedAt = Date() + try? await index.put(key: digest, descriptor: entry.descriptor, metadata: updatedMetadata) + + return result + } + + public func put(_ result: CachedResult, key: CacheKey, for operation: ContainerBuildIR.Operation) async { + // Generate cache digest + guard let digest = try? generateCacheDigest(from: key) else { + return + } + + // Check if already exists + if let _ = try? await index.get(key: digest) { + return + } + + do { + // Start new ingest session + let (sessionId, ingestDir) = try await contentStore.newIngestSession() + + do { + // Create content writer for this session + let writer = try ContentWriter(for: ingestDir) + + // Store components + let snapshotLayer = try await storeSnapshot(result.snapshot, using: writer) + let environmentLayer = try await storeEnvironment(result.environmentChanges, using: writer) + let metadataLayer = try await storeMetadata(result.metadataChanges, using: writer) + + // Create manifest + let manifest = createManifest( + key: key, + operation: operation, + layers: [snapshotLayer, environmentLayer, metadataLayer].compactMap { $0 } + ) + + // Write manifest + let (manifestSize, manifestDigest) = try writer.create(from: manifest) + + // Complete ingest session + _ = try await contentStore.completeIngestSession(sessionId) + + // Calculate total size + let _ = + manifestSize + + [snapshotLayer, environmentLayer, metadataLayer] + .compactMap { $0 } + .reduce(0) { $0 + $1.descriptor.size } + + // Record in index + let descriptor = Descriptor( + mediaType: manifest.mediaType, + digest: manifestDigest.digestString, + size: manifestSize + ) + + let operationHash: String + do { + operationHash = try operation.contentDigest().stringValue + } catch { + // If we can't compute the operation hash, use a fallback + operationHash = "unknown" + } + + let metadata = CacheMetadata( + createdAt: Date(), + accessedAt: Date(), + operationHash: operationHash, + platform: key.platform, + ttl: configuration.defaultTTL, + tags: [:] + ) + + try await index.put(key: digest, descriptor: descriptor, metadata: metadata) + + // Trigger eviction if needed + Task { [weak self] in + await self?.checkAndEvict() + } + + } catch { + // Cancel ingest session on error + try? await contentStore.cancelIngestSession(sessionId) + throw error + } + } catch { + // Log error but don't propagate - caching should not fail builds + print("Cache put failed: \(error)") + } + } + + public func has(key: CacheKey) async -> Bool { + guard let digest = try? generateCacheDigest(from: key) else { + return false + } + return (try? await index.get(key: digest)) != nil + } + + public func evict(keys: [CacheKey]) async { + for key in keys { + guard let digest = try? generateCacheDigest(from: key) else { + continue + } + + if let entry = try? await index.get(key: digest) { + // Get manifest to find content digests + if let manifest: CacheManifest = try? await contentStore.get(digest: entry.descriptor.digest) { + let digests = [entry.descriptor.digest] + manifest.allContentDigests() + _ = try? await contentStore.delete(digests: digests) + } + + // Remove from index + try? await index.remove(keys: [digest]) + } + } + } + + public func statistics() async -> CacheStatistics { + let stats = (try? await index.statistics()) ?? CacheStatistics.empty + + return CacheStatistics( + entryCount: stats.entryCount, + totalSize: stats.totalSize, + hitRate: stats.hitRate, + oldestEntryAge: stats.oldestEntryAge, + mostRecentEntryAge: stats.mostRecentEntryAge, + evictionPolicy: "lru", + compressionRatio: 1.0, + averageEntrySize: stats.entryCount > 0 ? stats.totalSize / UInt64(stats.entryCount) : 0, + operationMetrics: .empty, + errorCount: 0, + lastGCTime: nil, + shardInfo: nil + ) + } + + // MARK: - Private Methods + + /// Generate a deterministic cache digest from the cache key. + /// - Throws: CacheError.encodingFailed if UTF-8 encoding fails + private func generateCacheDigest(from key: CacheKey) throws -> String { + var hasher = SHA256() + + // Version prefix for cache invalidation + guard let versionData = configuration.cacheKeyVersion.data(using: .utf8) else { + // This should never happen as cacheKeyVersion is controlled internally + throw CacheError.encodingFailed("Failed to encode cache key version as UTF-8: \(configuration.cacheKeyVersion)") + } + hasher.update(data: versionData) + + // Operation digest + hasher.update(data: key.operationDigest.bytes) + + // Sorted input digests for determinism + for digest in key.inputDigests.sorted(by: { $0.stringValue < $1.stringValue }) { + hasher.update(data: digest.bytes) + } + + // Platform data + let platformData = encodePlatform(key.platform) + hasher.update(data: platformData) + + let digest = hasher.finalize() + let hexString = digest.map { String(format: "%02x", $0) }.joined() + return "sha256:\(hexString)" + } + + /// Encode platform to canonical form for hashing. + private func encodePlatform(_ platform: Platform) -> Data { + let normalized = NormalizedPlatform( + os: platform.os, + architecture: platform.architecture, + variant: platform.variant, + osVersion: platform.osVersion, + osFeatures: platform.osFeatures?.sorted() + ) + + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + return (try? encoder.encode(normalized)) ?? Data() + } + + /// Store snapshot data and return layer descriptor. + private func storeSnapshot(_ snapshot: Snapshot, using writer: ContentWriter) async throws -> CacheLayer? { + let data = try JSONEncoder().encode(snapshot) + let compressed = try compress(data) + let (size, digest) = try writer.write(compressed) + + let descriptor = Descriptor.forCacheContent( + mediaType: "application/vnd.container-build.snapshot.v1+json", + digest: digest.digestString, + size: size, + compressed: true, + annotations: [ + "com.apple.container-build.compression": configuration.compression.algorithm.rawValue, + "com.apple.container-build.uncompressed-size": String(data.count), + ] + ) + + return CacheLayer(descriptor: descriptor, type: .snapshot) + } + + /// Store environment changes and return layer descriptor. + private func storeEnvironment(_ changes: [String: EnvironmentValue], using writer: ContentWriter) async throws -> CacheLayer? { + guard !changes.isEmpty else { return nil } + + let data = try JSONEncoder().encode(changes) + let compressed = try compress(data) + let (size, digest) = try writer.write(compressed) + + let descriptor = Descriptor.forCacheContent( + mediaType: "application/vnd.container-build.environment.v1+json", + digest: digest.digestString, + size: size, + compressed: true, + annotations: [ + "com.apple.container-build.compression": configuration.compression.algorithm.rawValue + ] + ) + + return CacheLayer(descriptor: descriptor, type: .environment) + } + + /// Store metadata changes and return layer descriptor. + private func storeMetadata(_ changes: [String: String], using writer: ContentWriter) async throws -> CacheLayer? { + guard !changes.isEmpty else { return nil } + + let data = try JSONEncoder().encode(changes) + let compressed = try compress(data) + let (size, digest) = try writer.write(compressed) + + let descriptor = Descriptor.forCacheContent( + mediaType: "application/vnd.container-build.metadata.v1+json", + digest: digest.digestString, + size: size, + compressed: true, + annotations: [ + "com.apple.container-build.compression": configuration.compression.algorithm.rawValue + ] + ) + + return CacheLayer(descriptor: descriptor, type: .metadata) + } + + /// Create cache manifest. + private func createManifest(key: CacheKey, operation: ContainerBuildIR.Operation, layers: [CacheLayer]) -> CacheManifest { + CacheManifest( + schemaVersion: 2, + mediaType: CacheManifest.manifestMediaType, + config: CacheConfig( + cacheKey: SerializedCacheKey(from: key), + operationType: String(describing: type(of: operation)), + platform: key.platform, + buildVersion: "1.0.0" + ), + layers: layers, + annotations: [ + "com.apple.container-build.created": ISO8601DateFormatter().string(from: Date()), + "com.apple.container-build.cache-version": configuration.cacheKeyVersion, + ], + subject: nil + ) + } + + /// Reconstruct cached result from manifest. + private func reconstructResult(from manifest: CacheManifest) async throws -> CachedResult { + var snapshot: Snapshot? + var environmentChanges: [String: EnvironmentValue] = [:] + var metadataChanges: [String: String] = [:] + + for layer in manifest.layers { + guard let content = try await contentStore.get(digest: layer.descriptor.digest) else { + throw CacheError.storageFailed( + path: layer.descriptor.digest, underlyingError: NSError(domain: "Cache", code: 404, userInfo: [NSLocalizedDescriptionKey: "Missing content"])) + } + + let data = try content.data() + let decompressed = try decompress(data) + + switch layer.type { + case .snapshot: + snapshot = try JSONDecoder().decode(Snapshot.self, from: decompressed) + case .environment: + environmentChanges = try JSONDecoder().decode([String: EnvironmentValue].self, from: decompressed) + case .metadata: + metadataChanges = try JSONDecoder().decode([String: String].self, from: decompressed) + } + } + + guard let snapshot = snapshot else { + throw CacheError.storageFailed( + path: "Missing snapshot layer", underlyingError: NSError(domain: "Cache", code: 500, userInfo: [NSLocalizedDescriptionKey: "Invalid Manifest"])) + } + + return CachedResult( + snapshot: snapshot, + environmentChanges: environmentChanges, + metadataChanges: metadataChanges + ) + } + + /// Compress data based on configuration. + private func compress(_ data: Data) throws -> Data { + guard data.count >= configuration.compression.minSize else { + return data + } + + switch configuration.compression.algorithm { + case .none: + return data + case .zstd: + // Use zstd compression (would need actual implementation) + return data // Placeholder + case .lz4: + // Use lz4 compression (would need actual implementation) + return data // Placeholder + case .gzip: + // Use gzip compression + return try (data as NSData).compressed(using: .zlib) as Data + } + } + + /// Decompress data based on manifest metadata. + private func decompress(_ data: Data) throws -> Data { + // Check if data is compressed by trying to decompress + if let decompressed = try? (data as NSData).decompressed(using: .zlib) as Data { + return decompressed + } + return data + } + + /// Check if eviction is needed and trigger it. + private func checkAndEvict() async { + let stats = try? await index.statistics() + + guard let stats = stats else { return } + + // Check if we need to evict based on size + if stats.totalSize > configuration.maxSize { + await performEviction(targetSize: UInt64(Double(configuration.maxSize) * 0.8)) + } + } + + /// Perform cache eviction to reach target size. + private func performEviction(targetSize: UInt64) async { + let stats = try? await index.statistics() + guard let currentSize = stats?.totalSize, currentSize > targetSize else { return } + + let sizeToEvict = currentSize - targetSize + + // Get all entries and sort by access time (LRU) + guard let allEntries = try? await index.allEntries() else { return } + let sortedEntries = allEntries.sorted { $0.value.metadata.accessedAt < $1.value.metadata.accessedAt } + + var evictedSize: UInt64 = 0 + var keysToEvict: [String] = [] + var digestsToDelete: [String] = [] + + for (key, entry) in sortedEntries { + // Get manifest to find all content digests + if let manifest: CacheManifest = try? await contentStore.get(digest: entry.descriptor.digest) { + digestsToDelete.append(entry.descriptor.digest) + digestsToDelete.append(contentsOf: manifest.allContentDigests()) + } + + keysToEvict.append(key) + evictedSize += UInt64(entry.descriptor.size) + + if evictedSize >= sizeToEvict { + break + } + } + + // Remove from index + try? await index.remove(keys: keysToEvict) + + // Delete content from store + _ = try? await contentStore.delete(digests: digestsToDelete) + } + + /// Run periodic eviction task. + private func runPeriodicEviction() async { + while !Task.isCancelled { + // Sleep for GC interval + try? await Task.sleep(nanoseconds: UInt64(configuration.gcInterval * 1_000_000_000)) + + // Remove expired entries based on TTL + guard let allEntries = try? await index.allEntries() else { return } + var keysToEvict: [String] = [] + var digestsToDelete: [String] = [] + + for (key, entry) in allEntries { + // Check TTL + if let ttl = entry.metadata.ttl { + let expirationDate = entry.metadata.createdAt.addingTimeInterval(ttl) + if Date() > expirationDate { + keysToEvict.append(key) + + // Get manifest to find all content digests + if let manifest: CacheManifest = try? await contentStore.get(digest: entry.descriptor.digest) { + digestsToDelete.append(entry.descriptor.digest) + digestsToDelete.append(contentsOf: manifest.allContentDigests()) + } + } + } + } + + if !keysToEvict.isEmpty { + try? await index.remove(keys: keysToEvict) + _ = try? await contentStore.delete(digests: digestsToDelete) + } + + // Check size limits + await checkAndEvict() + } + } +} + +// MARK: - Extensions + +extension Platform { + var canonicalString: String { + var parts = ["\(os)/\(architecture)"] + if let variant = variant { + parts.append(variant) + } + return parts.joined(separator: "/") + } + + /// - Throws: CacheError.encodingFailed if UTF-8 encoding fails + func canonicalRepresentation() throws -> Data { + guard let data = canonicalString.data(using: .utf8) else { + // This should never happen as canonicalString contains only valid UTF-8 + throw CacheError.encodingFailed("Failed to encode canonical platform string as UTF-8: \(canonicalString)") + } + return data + } +} diff --git a/Sources/NativeBuilder/ContainerBuildDemo/Demo.swift b/Sources/NativeBuilder/ContainerBuildDemo/Demo.swift new file mode 100644 index 00000000..5360cd7e --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildDemo/Demo.swift @@ -0,0 +1,81 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildCache +import ContainerBuildExecutor +import ContainerBuildIR +import ContainerBuildReporting +import ContainerBuildSnapshotter +import Foundation + +/// A simple demonstration of the build execution system. +public struct Demo { + public static func runDemo() async throws { + // Set up the build environment first + let snapshotter = MemorySnapshotter() + let cache = MemoryBuildCache() + let reporter = Reporter() + + // Create a build graph with parallel operations, passing the reporter + let graph = try IRExample.createParallelBuild(reporter: reporter) + let config = Scheduler.Configuration( + maxConcurrency: ProcessInfo.processInfo.activeProcessorCount, + failFast: true, + enableProgressReporting: true + ) + let executor = Scheduler( + snapshotter: snapshotter, + cache: cache, + reporter: reporter, + configuration: config + ) + + // Create progress consumer + let consumer = PlainProgressConsumer( + configuration: .init() + ) + + // Start progress monitoring task + let progressTask = Task { + try await consumer.consume(reporter: reporter) + } + + // Register completion handler to wait for progress task + executor.onCompletion { + try? await progressTask.value + } + + // Execute the build - this will wait for progress to complete before returning + _ = try await executor.execute(graph) + + // Get build statistics + let stats = consumer.getStatistics() + + // Print build summary based on statistics + if let duration = stats.duration { + if stats.success == true { + print("\nBuild completed successfully in \(String(format: "%.2f", duration))s") + // print(" Total operations: \(stats.totalOperations)") + // print(" Cache hits: \(stats.cacheHits)") + // print(" Executed: \(stats.executedOperations)") + } else { + print("\nBuild failed after \(String(format: "%.2f", duration))s") + // print(" Failed operations: \(stats.failedOperations)") + } + } + } + +} diff --git a/Sources/NativeBuilder/ContainerBuildDemo/ErrorPresenter.swift b/Sources/NativeBuilder/ContainerBuildDemo/ErrorPresenter.swift new file mode 100644 index 00000000..fb831073 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildDemo/ErrorPresenter.swift @@ -0,0 +1,72 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildCache +import ContainerBuildExecutor +import ContainerBuildIR +import Foundation + +/// A utility to present various error types in a uniform, user-friendly format. +public struct ErrorPresenter { + /// Renders any error into a formatted string for console output. + public func present(error: Error) -> String { + var output: [String] = [] + output.append("❌ Build failed.") + output.append("----------------------------------------") + appendErrorDetails(error, to: &output, isRoot: true) + output.append("----------------------------------------") + return output.joined(separator: "\n") + } + + /// A recursive helper to print error chains. + private func appendErrorDetails(_ error: Error, to output: inout [String], isRoot: Bool) { + let title: String + var details: String = error.localizedDescription + var underlyingError: (any Error)? + + switch error { + case let err as BuildDefinitionError: + title = "Build Definition Error" + details = err.errorDescription ?? "No details." + + case let err as CacheError: + title = "Cache Error" + details = err.errorDescription ?? "No details." + if case .storageFailed(_, let underlying) = err { + underlyingError = underlying + } else if case .manifestUnreadable(_, let underlying) = err { + underlyingError = underlying + } + + case let err as ExecutorError: + title = "Build Execution Error" + // Extract richer details from the context + details = "Operation '\(String(describing: err.context.operation))' failed." + underlyingError = err.context.underlyingError + + default: + title = "Unexpected Error" + } + + output.append(isRoot ? "Reason: \(title)" : "Caused by: \(title)") + output.append(" ↳ Details: \(details)") + + // If there's a wrapped error, recurse. + if let underlyingError = underlyingError { + appendErrorDetails(underlyingError, to: &output, isRoot: false) + } + } +} diff --git a/Sources/NativeBuilder/ContainerBuildDemo/Examples.swift b/Sources/NativeBuilder/ContainerBuildDemo/Examples.swift new file mode 100644 index 00000000..224f2999 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildDemo/Examples.swift @@ -0,0 +1,327 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildReporting +import Foundation + +/// Example demonstrating how to build an IR graph programmatically. +/// +/// This example creates a multi-stage Node.js application build. +public enum IRExample { + + /// Create a simple single-stage build. + public static func createSimpleBuild() throws -> BuildGraph { + guard let baseImage = ImageReference(parsing: "ubuntu:22.04") else { + throw ReferenceError.invalidFormat("ubuntu:22.04") + } + + return try GraphBuilder.singleStage( + from: baseImage, + platform: .linuxAMD64 + ) { builder in + try builder + .run("apt-get update && apt-get install -y curl") + .workdir("/app") + .copyFromContext(paths: ["package.json", "src/"], to: "/app/") + .run("npm install") + .env("NODE_ENV", "production") + .expose(3000) + .cmd(Command.exec(["node", "src/index.js"])) + } + } + + /// Create a realistic multi-stage build showing actual parallelism + public static func createParallelBuild(reporter: Reporter? = nil) throws -> BuildGraph { + // In container builds, true parallelism happens between stages or with independent resources + + guard let nodeImage = ImageReference(parsing: "node:18-alpine"), + let goImage = ImageReference(parsing: "golang:1.21-alpine"), + let alpineImage = ImageReference(parsing: "alpine:3.18") + else { + throw ReferenceError.invalidFormat("Invalid image reference") + } + + return try GraphBuilder.multiStage(reporter: reporter) { builder in + // Stage 1: Build frontend assets + try builder + .stage(name: "frontend-builder", from: nodeImage) + .workdir("/frontend") + .copyFromContext(paths: ["frontend/package*.json"], to: "./") + .run("npm ci") + .copyFromContext(paths: ["frontend/"], to: "./") + .run("npm run build") + + // Stage 2: Build backend + try builder + .stage(name: "backend-builder", from: goImage) + .workdir("/backend") + .copyFromContext(paths: ["go.mod", "go.sum"], to: "./") + .run("go mod download") + .copyFromContext(paths: ["*.go", "cmd/", "internal/"], to: "./") + .run("CGO_ENABLED=0 go build -o server ./cmd/server") + + // Stage 3: Runtime - depends on both builders + try builder + .stage(name: "runtime", from: alpineImage) + .copyFromStage(.named("frontend-builder"), paths: ["/frontend/dist"], to: "/app/static") + .copyFromStage(.named("backend-builder"), paths: ["/backend/server"], to: "/app/server") + .run("chmod +x /app/server") + .expose(8080) + .cmd(.exec(["/app/server"])) + } + } + + /// Create a multi-stage build for a Go application. + public static func createMultiStageBuild() throws -> BuildGraph { + try GraphBuilder.multiStage { builder in + // Build stage + guard let builderImage = ImageReference(parsing: "golang:1.21-alpine") else { + throw ReferenceError.invalidFormat("golang:1.21-alpine") + } + + try builder + .stage( + name: "builder", + from: builderImage + ) + .workdir("/build") + .copyFromContext(paths: ["go.mod", "go.sum"], to: "./") + .run("go mod download") + .copyFromContext(paths: ["*.go"], to: "./") + .run("CGO_ENABLED=0 go build -o app") + + // Runtime stage + try builder + .scratch(name: "runtime") + .copyFromStage( + .named("builder"), + paths: ["/build/app"], + to: "/app", + chmod: .mode(0o755) + ) + .copyFromStage( + .named("builder"), + paths: ["/etc/ssl/certs/ca-certificates.crt"], + to: "/etc/ssl/certs/" + ) + .user(.uid(1000)) + .entrypoint(.exec(["/app"])) + } + } + + /// Create a Python application with best practices. + public static func createPythonBuild() throws -> BuildGraph { + let graph = try BuildGraph( + stages: [ + // Dependencies stage + BuildStage( + name: "dependencies", + base: { + guard let pythonImage = ImageReference(parsing: "python:3.11-slim") else { + throw ReferenceError.invalidFormat("python:3.11-slim") + } + return ImageOperation( + source: .registry(pythonImage), + platform: .linuxAMD64 + ) + }(), + nodes: [ + BuildNode( + operation: MetadataOperation( + action: .setWorkdir(path: "/app") + ) + ), + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .context(ContextSource(paths: ["requirements.txt"])), + destination: "/app/" + ) + ), + BuildNode( + operation: ExecOperation( + command: .shell("pip install --user --no-cache-dir -r requirements.txt") + ) + ), + ] + ), + + // Application stage + BuildStage( + name: "app", + base: { + guard let pythonImage = ImageReference(parsing: "python:3.11-slim") else { + throw ReferenceError.invalidFormat("python:3.11-slim") + } + return ImageOperation( + source: .registry(pythonImage), + platform: .linuxAMD64 + ) + }(), + nodes: [ + // Copy dependencies from first stage + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .stage(.named("dependencies"), paths: ["/root/.local"]), + destination: "/root/.local" + ) + ), + // Set up application + BuildNode( + operation: MetadataOperation( + action: .setWorkdir(path: "/app") + ) + ), + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .context(ContextSource(paths: ["*.py", "src/"])), + destination: "/app/" + ) + ), + // Configure runtime + BuildNode( + operation: MetadataOperation( + action: .setEnv( + key: "PYTHONPATH", + value: .literal("/root/.local/lib/python3.11/site-packages") + ) + ) + ), + BuildNode( + operation: MetadataOperation( + action: .setUser(user: .uidGid(uid: 1000, gid: 1000)) + ) + ), + BuildNode( + operation: MetadataOperation( + action: .expose(port: PortSpec(port: 8000)) + ) + ), + BuildNode( + operation: MetadataOperation( + action: .setHealthcheck( + healthcheck: Healthcheck( + test: .shell("curl -f http://localhost:8000/health || exit 1"), + interval: 30, + timeout: 3, + retries: 3 + ) + ) + ) + ), + BuildNode( + operation: MetadataOperation( + action: .setCmd(command: .exec(["python", "main.py"])) + ) + ), + ] + ), + ], + targetPlatforms: [.linuxAMD64, .linuxARM64] + ) + + return graph + } + + /// Demonstrate advanced features like cache mounts and secrets. + public static func createAdvancedBuild() throws -> BuildGraph { + guard let nodeImage = ImageReference(parsing: "node:18-alpine") else { + throw ReferenceError.invalidFormat("node:18-alpine") + } + + return try GraphBuilder.singleStage( + from: nodeImage + ) { builder in + try builder + .workdir("/app") + + // Cache mount for package manager + .run( + "npm ci", + mounts: [ + Mount( + type: .cache, + target: "/root/.npm", + options: MountOptions(sharing: .shared) + ) + ] + ) + + // Secret mount for private registry + .run( + "npm install @private/package", + mounts: [ + Mount( + type: .secret, + target: "/root/.npmrc", + source: .secret("npm-token"), + options: MountOptions(readOnly: true, mode: 0o600) + ) + ] + ) + + // Build with tmpfs for temporary files + .run( + "npm run build", + mounts: [ + Mount( + type: .tmpfs, + target: "/tmp", + options: MountOptions(size: 1024 * 1024 * 1024) // 1GB + ) + ] + ) + + // Multi-platform metadata + .label("org.opencontainers.image.source", "https://github.com/example/app") + .label("org.opencontainers.image.version", "${VERSION}") + .label("org.opencontainers.image.created", "${BUILD_DATE}") + } + } + + /// Validate and analyze a build graph. + public static func analyzeGraph(_ graph: BuildGraph) { + // Validate + let validator = StandardValidator() + let validationResult = validator.validate(graph) + + print("Validation Results:") + print(" Errors: \(validationResult.errors.count)") + for error in validationResult.errors { + print(" - \(error)") + } + print(" Warnings: \(validationResult.warnings.count)") + for warning in validationResult.warnings { + print(" - \(warning.message)") + if let suggestion = warning.suggestion { + print(" Suggestion: \(suggestion)") + } + } + + // Analyze with reporter + print("\nSemantic Analysis:") + + // Graph statistics + let stats = graph.analyze() + print("\nGraph Statistics:") + print(" Stages: \(stats.stageCount)") + print(" Operations: \(stats.operationCount)") + print(" Critical Path: \(stats.criticalPathLength) operations") + } +} diff --git a/Sources/NativeBuilder/ContainerBuildDemo/GraphVisualizer.swift b/Sources/NativeBuilder/ContainerBuildDemo/GraphVisualizer.swift new file mode 100644 index 00000000..765d1cf4 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildDemo/GraphVisualizer.swift @@ -0,0 +1,421 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildExecutor +import ContainerBuildIR +import ContainerBuildReporting +import Foundation + +/// Visualizes build graphs in various formats +public struct GraphVisualizer { + + // MARK: - ASCII Visualization + + /// Generate an ASCII representation of the build graph + public static func generateASCII(_ graph: BuildGraph) -> String { + var output = "" + let analysis = graph.analyze() + + // Header + output += "Build Graph\n" + output += "===========\n" + output += "Stages: \(analysis.stageCount) | Operations: \(analysis.operationCount) | Critical Path: \(analysis.criticalPathLength)\n" + output += "\n" + + // Process each stage and collect cross-stage dependencies + var crossStageDeps: [(from: String, to: String, desc: String)] = [] + + for (stageIndex, stage) in graph.stages.enumerated() { + let stageName = stage.name ?? "stage-\(stageIndex)" + output += drawStage(stage, name: stageName, in: graph) + + // Check for cross-stage dependencies + for node in stage.nodes { + if let fsOp = node.operation as? FilesystemOperation { + switch fsOp.source { + case .stage(let ref, let paths): + let sourceStage: String + switch ref { + case .named(let name): + sourceStage = name + case .index(let idx): + sourceStage = "stage-\(idx)" + case .previous: + sourceStage = stageIndex > 0 ? (graph.stages[stageIndex - 1].name ?? "stage-\(stageIndex-1)") : "unknown" + } + crossStageDeps.append((from: sourceStage, to: stageName, desc: "COPY \(paths.first ?? "")")) + default: + break + } + } + } + + output += "\n" + } + + // Show cross-stage dependencies + if !crossStageDeps.isEmpty { + output += "Cross-Stage Dependencies:\n" + for dep in crossStageDeps { + output += " \(dep.from) → \(dep.to) [\(dep.desc)]\n" + } + output += "\n" + } + + // Parallelism analysis + let parallelGroups = analysis.parallelismOpportunities.filter { $0.count > 1 } + if !parallelGroups.isEmpty { + output += "Parallelism Analysis\n" + output += "===================\n" + for (index, group) in parallelGroups.enumerated() { + output += "Group \(index + 1): \(group.count) operations can run in parallel\n" + } + } + + return output + } + + private static func drawStage(_ stage: BuildStage, name: String, in graph: BuildGraph) -> String { + var output = "" + + // Stage header + let headerLine = "┌─ Stage: \(name) " + output += headerLine + String(repeating: "─", count: max(50 - headerLine.count, 3)) + "┐\n" + + // First draw the base image operation + let baseDesc = formatNodeDescription(BuildNode(operation: stage.base)) + output += "│ [\(baseDesc)]\n" + + // Draw nodes in order + for (_, node) in stage.nodes.enumerated() { + output += "│\n" + output += "│ ↓\n" + output += drawNodesAtLevel([node], in: stage, showParallel: false) + } + + // Stage footer + output += "└" + String(repeating: "─", count: 51) + "┘\n" + + return output + } + + private static func computeNodeLevels(_ nodes: [BuildNode]) -> [[BuildNode]] { + var levels: [[BuildNode]] = [] + var nodeLevel: [UUID: Int] = [:] + + func computeLevel(for node: BuildNode) -> Int { + if let level = nodeLevel[node.id] { + return level + } + + var maxDepLevel = -1 + for depId in node.dependencies { + if let depNode = nodes.first(where: { $0.id == depId }) { + maxDepLevel = max(maxDepLevel, computeLevel(for: depNode)) + } + } + + let level = maxDepLevel + 1 + nodeLevel[node.id] = level + return level + } + + // Compute level for each node + for node in nodes { + let level = computeLevel(for: node) + while levels.count <= level { + levels.append([]) + } + levels[level].append(node) + } + + return levels + } + + private static func drawConnections(from previousNodes: [BuildNode], to currentNodes: [BuildNode], in stage: BuildStage) -> String { + var output = "│\n" + + // Check if any connections exist + var hasConnections = false + for node in currentNodes { + if !node.dependencies.isEmpty { + hasConnections = true + break + } + } + + if hasConnections { + // Draw connection lines + var connectionLine = "│ " + for (index, node) in currentNodes.enumerated() { + if index > 0 { + connectionLine += " " + } + + let depCount = node.dependencies.count + if depCount > 0 { + connectionLine += "╱" + if depCount > 1 { + connectionLine += "─┴─" + } else { + connectionLine += "───" + } + connectionLine += "╲" + } else { + connectionLine += " " + } + } + output += connectionLine + "\n" + } + + return output + } + + private static func drawNodesAtLevel(_ nodes: [BuildNode], in stage: BuildStage, showParallel: Bool) -> String { + var output = "│ " + + // Draw nodes + for (index, node) in nodes.enumerated() { + if index > 0 { + output += " " + if showParallel { + output += "║ " // Double bar indicates parallel execution + } else { + output += " " + } + } + + let desc = formatNodeDescription(node) + output += "[\(desc)]" + } + + output += "\n" + return output + } + + private static func formatNodeDescription(_ node: BuildNode) -> String { + let fullDesc = ReportContext.describeOperation(node.operation) + let parts = fullDesc.split(separator: " ", maxSplits: 1) + let operation = String(parts[0]) + let args = parts.count > 1 ? String(parts[1]) : "" + + // Format based on operation type + switch operation { + case "FROM": + return "FROM \(truncate(args, to: 20))" + case "RUN": + return "RUN \(truncate(args, to: 25))" + case "COPY", "ADD": + let paths = args.split(separator: " ") + let source = paths.first ?? "" + return "\(operation) \(truncate(String(source), to: 15))" + case "WORKDIR": + return "WORKDIR \(args)" + case "ENV": + return "ENV \(truncate(args, to: 20))" + case "EXPOSE": + return "EXPOSE \(args)" + case "CMD", "ENTRYPOINT": + return "\(operation) \(truncate(args, to: 18))" + case "LABEL": + return "LABEL \(truncate(args, to: 18))" + case "USER": + return "USER \(args)" + case "ARG": + return "ARG \(truncate(args, to: 20))" + case "VOLUME": + return "VOLUME \(args)" + default: + return "\(operation) \(truncate(args, to: 18))" + } + } + + private static func truncate(_ string: String, to length: Int) -> String { + if string.count <= length { + return string + } + return String(string.prefix(length - 3)) + "..." + } + + // MARK: - Graphviz DOT Format + + /// Generate a Graphviz DOT representation of the build graph + public static func generateDOT(_ graph: BuildGraph) -> String { + var output = "digraph BuildGraph {\n" + output += " rankdir=TB;\n" + output += " node [shape=box, style=rounded];\n" + output += " \n" + + // Graph metadata + output += " label=\"Build Graph - \(graph.stages.count) stages, \(graph.targetPlatforms.count) platforms\";\n" + output += " labelloc=t;\n" + output += " \n" + + // Process each stage + for (stageIndex, stage) in graph.stages.enumerated() { + let stageName = stage.name ?? "stage_\(stageIndex)" + + // Create subgraph for stage + output += " subgraph cluster_\(stageIndex) {\n" + output += " label=\"Stage: \(stageName)\";\n" + output += " style=dotted;\n" + output += " color=gray;\n" + output += " \n" + + // Add nodes + for node in stage.nodes { + let nodeId = sanitizeNodeId(node.id.uuidString) + let label = formatNodeLabel(node) + let color = getNodeColor(for: node.operation) + + output += " \(nodeId) [label=\"\(label)\", fillcolor=\"\(color)\", style=filled];\n" + } + + // Add dependencies within stage + for node in stage.nodes { + let nodeId = sanitizeNodeId(node.id.uuidString) + for depId in node.dependencies { + let depNodeId = sanitizeNodeId(depId.uuidString) + output += " \(depNodeId) -> \(nodeId);\n" + } + } + + output += " }\n" + output += " \n" + } + + // Add cross-stage dependencies + output += " // Cross-stage dependencies\n" + for (stageIndex, stage) in graph.stages.enumerated() { + for node in stage.nodes { + if let fsOp = node.operation as? FilesystemOperation { + switch fsOp.source { + case .stage(let ref, _): + if let sourceStage = graph.resolveStage(ref), + let sourceIndex = graph.stages.firstIndex(where: { $0.id == sourceStage.id }), + sourceIndex != stageIndex + { + // Draw edge from last node of source stage to this node + if let lastNode = sourceStage.nodes.last { + let sourceId = sanitizeNodeId(lastNode.id.uuidString) + let targetId = sanitizeNodeId(node.id.uuidString) + output += " \(sourceId) -> \(targetId) [style=dashed, color=blue, label=\"stage copy\"];\n" + } + } + default: + break + } + } + } + } + + output += "}\n" + return output + } + + private static func sanitizeNodeId(_ id: String) -> String { + "node_" + id.replacingOccurrences(of: "-", with: "_") + } + + private static func formatNodeLabel(_ node: BuildNode) -> String { + let desc = ReportContext.describeOperation(node.operation) + let parts = desc.split(separator: " ", maxSplits: 1) + let operation = String(parts[0]) + let args = parts.count > 1 ? String(parts[1]) : "" + + // Escape quotes for DOT format + let escapedArgs = + args + .replacingOccurrences(of: "\"", with: "\\\"") + .replacingOccurrences(of: "\n", with: "\\n") + + switch operation { + case "RUN", "COPY", "ADD", "CMD", "ENTRYPOINT": + return "\(operation)\\n\(truncate(escapedArgs, to: 30))" + default: + return "\(operation)\\n\(escapedArgs)" + } + } + + private static func getNodeColor(for operation: any ContainerBuildIR.Operation) -> String { + switch operation { + case is ImageOperation: + return "#E8F5E9" // Light green + case is ExecOperation: + return "#E3F2FD" // Light blue + case is FilesystemOperation: + return "#FFF3E0" // Light orange + case is MetadataOperation: + return "#F3E5F5" // Light purple + default: + return "#F5F5F5" // Light gray + } + } + + // MARK: - Mermaid Format + + /// Generate a Mermaid diagram representation of the build graph + public static func generateMermaid(_ graph: BuildGraph) -> String { + var output = "graph TB\n" + + // Process each stage + for (stageIndex, stage) in graph.stages.enumerated() { + let stageName = stage.name ?? "stage-\(stageIndex)" + + // Add stage label + output += " subgraph \(stageName)\n" + + // Add nodes + for node in stage.nodes { + let nodeId = "N" + node.id.uuidString.prefix(8) + let label = formatNodeDescription(node) + output += " \(nodeId)[\"\(label)\"]\n" + } + + // Add dependencies + for node in stage.nodes { + let nodeId = "N" + node.id.uuidString.prefix(8) + for depId in node.dependencies { + let depNodeId = "N" + depId.uuidString.prefix(8) + output += " \(depNodeId) --> \(nodeId)\n" + } + } + + output += " end\n" + } + + return output + } +} + +// MARK: - Convenience Extensions + +extension BuildGraph { + /// Generate ASCII visualization of this graph + public var asciiDiagram: String { + GraphVisualizer.generateASCII(self) + } + + /// Generate Graphviz DOT format of this graph + public var dotFormat: String { + GraphVisualizer.generateDOT(self) + } + + /// Generate Mermaid diagram of this graph + public var mermaidDiagram: String { + GraphVisualizer.generateMermaid(self) + } +} diff --git a/Sources/NativeBuilder/ContainerBuildDemo/main.swift b/Sources/NativeBuilder/ContainerBuildDemo/main.swift new file mode 100644 index 00000000..df028a0d --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildDemo/main.swift @@ -0,0 +1,20 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import Foundation + +try await Demo.runDemo() diff --git a/Sources/NativeBuilder/ContainerBuildDiffer/Differ.swift b/Sources/NativeBuilder/ContainerBuildDiffer/Differ.swift new file mode 100644 index 00000000..cb4c8eba --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildDiffer/Differ.swift @@ -0,0 +1,72 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildSnapshotter +import Foundation + +/// A protocol for computing differences between filesystem snapshots. +public protocol Differ: Sendable { + /// Compute the difference between two snapshots. + /// + /// - Parameters: + /// - from: The base snapshot + /// - to: The target snapshot + /// - Returns: The filesystem changes needed to transform `from` into `to` + func diff(from: Snapshot?, to: Snapshot) async throws -> FilesystemChanges + + /// Compute a digest representing the state of a filesystem path. + /// + /// - Parameter path: The filesystem path to digest + /// - Returns: A digest representing the current state + func digest(path: String) async throws -> Digest +} + +/// A basic in-memory differ implementation. +public struct MemoryDiffer: Differ { + public init() {} + + public func diff(from base: Snapshot?, to target: Snapshot) async throws -> FilesystemChanges { + // Stub implementation + // In a real implementation, this would: + // 1. Mount or access both snapshots + // 2. Walk the filesystem trees + // 3. Compare files, directories, and metadata + // 4. Return the differences + + FilesystemChanges( + added: Set(), + modified: Set(), + deleted: Set(), + sizeChange: 0 + ) + } + + public func digest(path: String) async throws -> Digest { + // Stub implementation + // In a real implementation, this would compute a merkle tree + // digest of the filesystem at the given path + + var digestBytes = Data(count: 32) + digestBytes.withUnsafeMutableBytes { bytes in + if let baseAddress = bytes.baseAddress { + memset(baseAddress, 0, 32) + } + } + + return try Digest(algorithm: .sha256, bytes: digestBytes) + } +} diff --git a/Sources/NativeBuilder/ContainerBuildExecutor/BuildExecutor.swift b/Sources/NativeBuilder/ContainerBuildExecutor/BuildExecutor.swift new file mode 100644 index 00000000..42b642a5 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildExecutor/BuildExecutor.swift @@ -0,0 +1,169 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildCache +import ContainerBuildIR +import ContainerizationOCI +import Foundation + +/// The main executor responsible for orchestrating container build execution. +/// +/// This protocol defines the high-level interface for executing build graphs. +/// Implementations coordinate stage execution, handle caching, and manage build state. +public protocol BuildExecutor: Sendable { + /// Execute a complete build graph. + /// + /// - Parameter graph: The build graph to execute + /// - Returns: The result of the build execution + /// - Throws: Any errors encountered during execution + func execute(_ graph: BuildGraph) async throws -> BuildResult + + /// Cancel any ongoing execution. + /// + /// This should gracefully stop execution and clean up resources. + func cancel() async +} + +/// The result of executing a build graph. +public struct BuildResult: Sendable { + /// The final image manifest for each target platform. + public let manifests: [Platform: ImageManifest] + + /// Execution metrics for performance analysis. + public let metrics: ExecutionMetrics + + /// Cache statistics for the build. + public let cacheStats: CacheStatistics + + /// Logs generated during the build. + public let logs: [String]? + + public init( + manifests: [Platform: ImageManifest], + metrics: ExecutionMetrics, + cacheStats: CacheStatistics, + logs: [String]? = nil + ) { + self.manifests = manifests + self.metrics = metrics + self.cacheStats = cacheStats + self.logs = logs + } +} + +/// Represents a built container image manifest. +public struct ImageManifest: Sendable { + /// The content-addressed digest of the image. + public let digest: Digest + + /// The size of the image in bytes. + public let size: Int64 + + /// The configuration digest. + public let configDigest: Digest + + /// Layer digests in order. + public let layers: [LayerDescriptor] + + public init( + digest: Digest, + size: Int64, + configDigest: Digest, + layers: [LayerDescriptor] + ) { + self.digest = digest + self.size = size + self.configDigest = configDigest + self.layers = layers + } +} + +/// Describes a single layer in an image. +public struct LayerDescriptor: Sendable { + /// The digest of the layer. + public let digest: Digest + + /// The size of the layer in bytes. + public let size: Int64 + + /// The media type of the layer. + public let mediaType: String + + public init(digest: Digest, size: Int64, mediaType: String = "application/vnd.oci.image.layer.v1.tar+gzip") { + self.digest = digest + self.size = size + self.mediaType = mediaType + } +} + +/// Metrics collected during build execution. +public struct ExecutionMetrics: Sendable { + /// Total execution time. + public let totalDuration: TimeInterval + + /// Time spent on each stage. + public let stageDurations: [String: TimeInterval] + + /// Number of operations executed. + public let operationCount: Int + + /// Number of operations that were cached. + public let cachedOperationCount: Int + + /// Total bytes transferred. + public let bytesTransferred: Int64 + + public init( + totalDuration: TimeInterval, + stageDurations: [String: TimeInterval], + operationCount: Int, + cachedOperationCount: Int, + bytesTransferred: Int64 + ) { + self.totalDuration = totalDuration + self.stageDurations = stageDurations + self.operationCount = operationCount + self.cachedOperationCount = cachedOperationCount + self.bytesTransferred = bytesTransferred + } +} + +/// Errors that can occur during build execution. +public enum BuildExecutorError: LocalizedError { + case stageNotFound(String) + case cyclicDependency + case operationFailed(ContainerBuildIR.Operation, underlying: Error) + case cancelled + case unsupportedOperation(ContainerBuildIR.Operation) + case internalError(String) + + public var errorDescription: String? { + switch self { + case .stageNotFound(let name): + return "Stage not found: '\(name)'" + case .cyclicDependency: + return "Cyclic dependency detected in build graph" + case .operationFailed(let op, let error): + return "Operation failed: \(op) - \(error.localizedDescription)" + case .cancelled: + return "Build execution was cancelled" + case .unsupportedOperation(let op): + return "Unsupported operation: \(type(of: op))" + case .internalError(let message): + return "Internal error: \(message)" + } + } +} diff --git a/Sources/NativeBuilder/ContainerBuildExecutor/ExecutionContext.swift b/Sources/NativeBuilder/ContainerBuildExecutor/ExecutionContext.swift new file mode 100644 index 00000000..50ab549c --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildExecutor/ExecutionContext.swift @@ -0,0 +1,248 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildReporting +import ContainerBuildSnapshotter +import ContainerizationOCI +import Foundation + +/// Carries execution state through operation execution. +/// +/// The context maintains the current state of the build, including filesystem +/// snapshots, environment variables, and other mutable state that operations +/// may read or modify. +public final class ExecutionContext: @unchecked Sendable { + /// The current build stage being executed. + public let stage: BuildStage + + /// The complete build graph. + public let graph: BuildGraph + + /// The target platform for this execution. + public let platform: Platform + + /// Progress reporter for build events. + public let reporter: Reporter + + /// Current environment variables. + private var _environment: Environment + + /// Current working directory. + private var _workingDirectory: String + + /// Current user. + private var _user: ContainerBuildIR.User? + + /// Image configuration being built. + private var _imageConfig: OCIImageConfig + + /// Snapshots for each executed node. + private var _snapshots: [UUID: Snapshot] + + /// Lock for thread-safe access. + private let lock = NSLock() + + public init( + stage: BuildStage, + graph: BuildGraph, + platform: Platform, + reporter: Reporter, + baseEnvironment: Environment = .init(), + baseConfig: OCIImageConfig? = nil + ) { + self.stage = stage + self.graph = graph + self.platform = platform + self.reporter = reporter + self._environment = baseEnvironment + self._workingDirectory = "/" + self._user = nil + self._imageConfig = baseConfig ?? OCIImageConfig(platform: platform) + self._snapshots = [:] + } + + /// Get the current environment. + public var environment: Environment { + lock.withLock { _environment } + } + + /// Update the environment. + public func updateEnvironment(_ updates: [String: EnvironmentValue]) { + lock.withLock { + // Create new environment with updates + var newVars = _environment.variables + for (key, value) in updates { + // Remove existing entries for this key + newVars.removeAll { $0.key == key } + // Add new entry + newVars.append((key: key, value: value)) + } + _environment = Environment(newVars) + } + } + + /// Get the current working directory. + public var workingDirectory: String { + lock.withLock { _workingDirectory } + } + + /// Set the working directory. + public func setWorkingDirectory(_ path: String) { + lock.withLock { _workingDirectory = path } + } + + /// Get the current user. + public var user: ContainerBuildIR.User? { + lock.withLock { _user } + } + + /// Set the current user. + public func setUser(_ user: ContainerBuildIR.User?) { + lock.withLock { _user = user } + } + + /// Get the current image configuration. + public var imageConfig: OCIImageConfig { + lock.withLock { _imageConfig } + } + + /// Update the image configuration. + public func updateImageConfig(_ updates: (inout OCIImageConfig) -> Void) { + lock.withLock { + updates(&_imageConfig) + } + } + + /// Get the snapshot for a node. + public func snapshot(for nodeId: UUID) -> Snapshot? { + lock.withLock { _snapshots[nodeId] } + } + + /// Set the snapshot for a node. + public func setSnapshot(_ snapshot: Snapshot, for nodeId: UUID) { + lock.withLock { _snapshots[nodeId] = snapshot } + } + + /// Get the latest snapshot (from the most recently executed node). + public func latestSnapshot() -> Snapshot? { + lock.withLock { + // In a real implementation, we'd track execution order + // For now, return any snapshot + _snapshots.values.first + } + } + + /// Create a child context for a nested execution. + public func childContext(for stage: BuildStage) -> ExecutionContext { + lock.withLock { + ExecutionContext( + stage: stage, + graph: graph, + platform: platform, + reporter: reporter, + baseEnvironment: Environment(_environment.variables), + baseConfig: _imageConfig + ) + } + } +} + +/// OCI image configuration. +/// +/// Represents the configuration for an OCI container image. +public struct OCIImageConfig: Sendable { + /// The platform this image is for. + public let platform: Platform + + /// Environment variables. + public var env: [String] + + /// Default command. + public var cmd: [String]? + + /// Entry point. + public var entrypoint: [String]? + + /// Working directory. + public var workingDir: String? + + /// User. + public var user: String? + + /// Exposed ports. + public var exposedPorts: Set + + /// Volumes. + public var volumes: Set + + /// Labels. + public var labels: [String: String] + + /// Stop signal. + public var stopSignal: String? + + /// Health check. + public var healthcheck: Healthcheck? + + public init( + platform: Platform, + env: [String] = [], + cmd: [String]? = nil, + entrypoint: [String]? = nil, + workingDir: String? = nil, + user: String? = nil, + exposedPorts: Set = [], + volumes: Set = [], + labels: [String: String] = [:], + stopSignal: String? = nil, + healthcheck: Healthcheck? = nil + ) { + self.platform = platform + self.env = env + self.cmd = cmd + self.entrypoint = entrypoint + self.workingDir = workingDir + self.user = user + self.exposedPorts = exposedPorts + self.volumes = volumes + self.labels = labels + self.stopSignal = stopSignal + self.healthcheck = healthcheck + } +} + +// Helper extension for thread-safe lock usage +extension NSLock { + func withLock(_ body: () throws -> T) rethrows -> T { + lock() + defer { unlock() } + return try body() + } +} + +// Helper extension for Environment +extension Environment { + /// Get the value for a key. + public func get(_ key: String) -> EnvironmentValue? { + for (k, v) in variables.reversed() { + if k == key { + return v + } + } + return nil + } +} diff --git a/Sources/NativeBuilder/ContainerBuildExecutor/ExecutionDispatcher.swift b/Sources/NativeBuilder/ContainerBuildExecutor/ExecutionDispatcher.swift new file mode 100644 index 00000000..32a96a65 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildExecutor/ExecutionDispatcher.swift @@ -0,0 +1,232 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerizationOCI +import Foundation + +/// Routes operations to appropriate executors based on capabilities and constraints. +/// +/// The dispatcher maintains a registry of executors and matches operations to +/// executors based on operation type, platform requirements, and executor capabilities. +public final class ExecutionDispatcher: Sendable { + /// Registered executors. + private let executors: [any OperationExecutor] + + /// Semaphores for concurrency control per executor. + private let semaphores: [ObjectIdentifier: AsyncSemaphore] + + public init(executors: [any OperationExecutor]) { + self.executors = executors + + // Create semaphores based on executor capabilities + var semas: [ObjectIdentifier: AsyncSemaphore] = [:] + for executor in executors { + let id = ObjectIdentifier(type(of: executor)) + semas[id] = AsyncSemaphore(value: executor.capabilities.maxConcurrency) + } + self.semaphores = semas + } + + /// Dispatch an operation to an appropriate executor. + /// + /// - Parameters: + /// - operation: The operation to execute + /// - context: The execution context + /// - constraints: Any additional constraints from the build node + /// - Returns: The execution result + /// - Throws: If no suitable executor is found or execution fails + public func dispatch( + _ operation: ContainerBuildIR.Operation, + context: ExecutionContext, + constraints: NodeConstraints? = nil + ) async throws -> ExecutionResult { + // Find a suitable executor + guard + let executor = findExecutor( + for: operation, + platform: context.platform, + constraints: constraints + ) + else { + throw BuildExecutorError.unsupportedOperation(operation) + } + + // Get semaphore for concurrency control + let executorId = ObjectIdentifier(type(of: executor)) + guard let semaphore = semaphores[executorId] else { + throw BuildExecutorError.internalError("Semaphore not found for executor \(executorId)") + } + + // Execute with concurrency limit + return try await semaphore.withPermit { + try await executor.execute(operation, context: context) + } + } + + /// Find an executor that can handle the given operation. + private func findExecutor( + for operation: ContainerBuildIR.Operation, + platform: Platform, + constraints: NodeConstraints? + ) -> (any OperationExecutor)? { + // Score each executor based on how well it matches + let candidates = executors.compactMap { executor -> (executor: any OperationExecutor, score: Int)? in + guard executor.canExecute(operation) else { return nil } + + let capabilities = executor.capabilities + var score = 0 + + // Check operation kind support + if capabilities.supportedOperations.contains(operation.operationKind) { + score += 100 + } + + // Check platform support + if let supportedPlatforms = capabilities.supportedPlatforms { + guard supportedPlatforms.contains(platform) else { + return nil // Platform not supported + } + score += 50 + } else { + score += 25 // Supports all platforms + } + + // Check privilege requirements + if let constraints = constraints, constraints.requiresPrivileged { + guard capabilities.requiresPrivileged else { + return nil // Cannot satisfy privilege requirement + } + score += 10 + } + + // Check resource requirements + if let constraints = constraints { + if !satisfiesResourceRequirements( + capabilities.resources, + constraints: constraints + ) { + return nil + } + } + + return (executor, score) + } + + // Return the highest scoring executor + return candidates.max(by: { $0.score < $1.score })?.executor + } + + /// Check if executor resources satisfy constraints. + private func satisfiesResourceRequirements( + _ resources: ResourceRequirements, + constraints: NodeConstraints + ) -> Bool { + // Check memory requirements + if let requiredMemory = constraints.minMemory, + let availableMemory = resources.minMemory, + availableMemory < requiredMemory + { + return false + } + + // Check disk requirements + if let requiredDisk = constraints.minDiskSpace, + let availableDisk = resources.minDiskSpace, + availableDisk < requiredDisk + { + return false + } + + // Check CPU architecture + if let requiredArch = constraints.cpuArchitecture, + let availableArch = resources.cpuArchitecture, + availableArch != requiredArch + { + return false + } + + return true + } +} + +/// Constraints that can be applied to node execution. +public struct NodeConstraints: Sendable { + /// Whether privileged execution is required. + public let requiresPrivileged: Bool + + /// Minimum memory required. + public let minMemory: Int64? + + /// Minimum disk space required. + public let minDiskSpace: Int64? + + /// Required CPU architecture. + public let cpuArchitecture: String? + + /// Custom constraints. + public let custom: [String: String] + + public init( + requiresPrivileged: Bool = false, + minMemory: Int64? = nil, + minDiskSpace: Int64? = nil, + cpuArchitecture: String? = nil, + custom: [String: String] = [:] + ) { + self.requiresPrivileged = requiresPrivileged + self.minMemory = minMemory + self.minDiskSpace = minDiskSpace + self.cpuArchitecture = cpuArchitecture + self.custom = custom + } +} + +/// A simple async semaphore for concurrency control. +actor AsyncSemaphore { + private var permits: Int + private var waiters: [CheckedContinuation] = [] + + init(value: Int) { + self.permits = value + } + + func acquire() async { + if permits > 0 { + permits -= 1 + return + } + + await withCheckedContinuation { continuation in + waiters.append(continuation) + } + } + + func release() { + if let waiter = waiters.first { + waiters.removeFirst() + waiter.resume() + } else { + permits += 1 + } + } + + func withPermit(_ body: () async throws -> T) async throws -> T { + await acquire() + defer { Task { release() } } + return try await body() + } +} diff --git a/Sources/NativeBuilder/ContainerBuildExecutor/ExecutorError.swift b/Sources/NativeBuilder/ContainerBuildExecutor/ExecutorError.swift new file mode 100644 index 00000000..301b9d4a --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildExecutor/ExecutorError.swift @@ -0,0 +1,51 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import Foundation + +/// An error originating from the build executor, enriched with runtime context. +public struct ExecutorError: Error, LocalizedError { + public let type: FailureType + public let context: ErrorContext + + public var errorDescription: String? { + "Execution failed: \(context.underlyingError.localizedDescription)" + } +} + +extension ExecutorError { + /// The general category of the execution failure. + public enum FailureType: Sendable { + case executionFailed + case cancelled + case invalidConfiguration + } + + /// Represents the detailed context of an error that occurred during a build. + public struct ErrorContext: Sendable { + public let operation: ContainerBuildIR.Operation // The operation that failed + public let underlyingError: any Error + public let diagnostics: Diagnostics + } + + /// Basic diagnostic information captured at failure time. + public struct Diagnostics: Sendable { + public let environment: [String: String] + public let workingDirectory: String + public let recentLogs: [String] + } +} diff --git a/Sources/NativeBuilder/ContainerBuildExecutor/Executors/ExecOperationExecutor.swift b/Sources/NativeBuilder/ContainerBuildExecutor/Executors/ExecOperationExecutor.swift new file mode 100644 index 00000000..c0007683 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildExecutor/Executors/ExecOperationExecutor.swift @@ -0,0 +1,105 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildSnapshotter +import Foundation + +/// Executes ExecOperation (RUN commands). +public struct ExecOperationExecutor: OperationExecutor { + public let capabilities: ExecutorCapabilities + + public init() { + self.capabilities = ExecutorCapabilities( + supportedOperations: [.exec], + maxConcurrency: 5 + ) + } + + public func execute(_ operation: ContainerBuildIR.Operation, context: ExecutionContext) async throws -> ExecutionResult { + guard let execOp = operation as? ExecOperation else { + throw ExecutorError( + type: .invalidConfiguration, + context: ExecutorError.ErrorContext( + operation: operation, underlyingError: NSError(domain: "Executor", code: 1, userInfo: [NSLocalizedDescriptionKey: "Unsupported operation"]), + diagnostics: ExecutorError.Diagnostics(environment: [:], workingDirectory: "", recentLogs: []))) + } + do { + // Stub implementation + // In a real implementation, this would: + // 1. Prepare the container environment + // 2. Execute the command + // 3. Capture output and changes + // 4. Update the snapshot + + let startTime = Date() + + // Simulate command execution + let commandString = execOp.command.displayString + let output = ExecutionOutput( + stdout: "Executing: \(commandString)\nOutput from command execution...\nDone.", + stderr: "", + exitCode: 0 + ) + + // Simulate filesystem changes + let changes = ContainerBuildSnapshotter.FilesystemChanges( + added: ["/tmp/exec-\(UUID().uuidString)"], + sizeChange: 1024 + ) + + // Create a new snapshot + let snapshot = try ContainerBuildSnapshotter.Snapshot( + digest: Digest(algorithm: .sha256, bytes: Data(count: 32)), + size: 1024, + parent: context.latestSnapshot()?.id + ) + + let duration = Date().timeIntervalSince(startTime) + + return ExecutionResult( + filesystemChanges: changes, + environmentChanges: [:], + metadataChanges: [:], + snapshot: snapshot, + duration: duration, + output: output + ) + } catch { + // Collect diagnostics + let environment = context.environment.effectiveEnvironment + + let diagnostics = ExecutorError.Diagnostics( + environment: environment, + workingDirectory: context.workingDirectory, + recentLogs: ["Failed to execute: \(execOp.command.displayString)"] + ) + + throw ExecutorError( + type: .executionFailed, + context: ExecutorError.ErrorContext( + operation: operation, + underlyingError: error, + diagnostics: diagnostics + ) + ) + } + } + + public func canExecute(_ operation: ContainerBuildIR.Operation) -> Bool { + operation is ExecOperation + } +} diff --git a/Sources/NativeBuilder/ContainerBuildExecutor/Executors/FilesystemOperationExecutor.swift b/Sources/NativeBuilder/ContainerBuildExecutor/Executors/FilesystemOperationExecutor.swift new file mode 100644 index 00000000..ac0233b3 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildExecutor/Executors/FilesystemOperationExecutor.swift @@ -0,0 +1,101 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildSnapshotter +import Foundation + +/// Executes FilesystemOperation (COPY, ADD, etc.). +public struct FilesystemOperationExecutor: OperationExecutor { + public let capabilities: ExecutorCapabilities + + public init() { + self.capabilities = ExecutorCapabilities( + supportedOperations: [.filesystem], + maxConcurrency: 10 + ) + } + + public func execute(_ operation: ContainerBuildIR.Operation, context: ExecutionContext) async throws -> ExecutionResult { + guard let fsOp = operation as? FilesystemOperation else { + throw ExecutorError( + type: .invalidConfiguration, + context: ExecutorError.ErrorContext( + operation: operation, underlyingError: NSError(domain: "Executor", code: 1), + diagnostics: ExecutorError.Diagnostics(environment: [:], workingDirectory: "", recentLogs: []))) + } + + do { + // Stub implementation + // In a real implementation, this would: + // 1. Resolve the source (context, stage, URL) + // 2. Copy/add/remove files as specified + // 3. Apply file metadata (permissions, ownership) + // 4. Update the snapshot + + let startTime = Date() + + // Simulate filesystem changes based on action + let changes: ContainerBuildSnapshotter.FilesystemChanges + switch fsOp.action { + case .copy, .add: + changes = ContainerBuildSnapshotter.FilesystemChanges( + added: [fsOp.destination], + sizeChange: 4096 + ) + case .remove: + changes = ContainerBuildSnapshotter.FilesystemChanges( + deleted: [fsOp.destination], + sizeChange: -1024 + ) + case .mkdir: + changes = ContainerBuildSnapshotter.FilesystemChanges( + added: [fsOp.destination], + sizeChange: 0 + ) + case .symlink, .hardlink: + changes = ContainerBuildSnapshotter.FilesystemChanges( + added: [fsOp.destination], + sizeChange: 0 + ) + } + + // Create a new snapshot + let snapshot = try ContainerBuildSnapshotter.Snapshot( + digest: Digest(algorithm: .sha256, bytes: Data(count: 32)), + size: 4096, + parent: context.latestSnapshot()?.id + ) + + let duration = Date().timeIntervalSince(startTime) + + return ExecutionResult( + filesystemChanges: changes, + snapshot: snapshot, + duration: duration + ) + } catch { + throw ExecutorError( + type: .executionFailed, + context: ExecutorError.ErrorContext( + operation: operation, underlyingError: error, diagnostics: ExecutorError.Diagnostics(environment: [:], workingDirectory: "", recentLogs: []))) + } + } + + public func canExecute(_ operation: ContainerBuildIR.Operation) -> Bool { + operation is FilesystemOperation + } +} diff --git a/Sources/NativeBuilder/ContainerBuildExecutor/Executors/ImageOperationExecutor.swift b/Sources/NativeBuilder/ContainerBuildExecutor/Executors/ImageOperationExecutor.swift new file mode 100644 index 00000000..de1cc3ee --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildExecutor/Executors/ImageOperationExecutor.swift @@ -0,0 +1,136 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildSnapshotter +import Foundation + +/// Executes ImageOperation (FROM instructions). +public struct ImageOperationExecutor: OperationExecutor { + public let capabilities: ExecutorCapabilities + + public init() { + self.capabilities = ExecutorCapabilities( + supportedOperations: [.image], + maxConcurrency: 3 + ) + } + + public func execute(_ operation: ContainerBuildIR.Operation, context: ExecutionContext) async throws -> ExecutionResult { + guard let imageOp = operation as? ImageOperation else { + throw ExecutorError( + type: .invalidConfiguration, + context: ExecutorError.ErrorContext( + operation: operation, underlyingError: NSError(domain: "Executor", code: 1), + diagnostics: ExecutorError.Diagnostics(environment: [:], workingDirectory: "", recentLogs: []))) + } + + do { + // Stub implementation + // In a real implementation, this would: + // 1. Pull the image from registry (if needed) + // 2. Verify the image (if verification specified) + // 3. Extract the image filesystem + // 4. Create initial snapshot + + let startTime = Date() + + // Simulate image pull + let imageSize: Int64 + let imageDigest: Digest + + switch imageOp.source { + case .registry(let reference): + // Simulate pulling from registry + imageSize = 100 * 1024 * 1024 // 100MB + let fakeDataString = "fake-image-\(reference.stringValue)" + guard let fakeData = fakeDataString.data(using: .utf8) else { + throw ExecutorError( + type: .executionFailed, + context: ExecutorError.ErrorContext( + operation: operation, + underlyingError: NSError(domain: "ImageOperationExecutor", code: 1, userInfo: [NSLocalizedDescriptionKey: "Failed to encode fake image data as UTF-8"]), + diagnostics: ExecutorError.Diagnostics(environment: [:], workingDirectory: "", recentLogs: []) + ) + ) + } + var digestBytes = Data(count: 32) + fakeData.withUnsafeBytes { bytes in + digestBytes.withUnsafeMutableBytes { digestBytesPtr in + if let destBase = digestBytesPtr.baseAddress, let srcBase = bytes.baseAddress { + memcpy(destBase, srcBase, min(32, bytes.count)) + } + } + } + imageDigest = try Digest(algorithm: .sha256, bytes: digestBytes) + + case .scratch: + // Empty image + imageSize = 0 + imageDigest = try Digest(algorithm: .sha256, bytes: Data(count: 32)) + + case .ociLayout: + // Simulate loading from OCI layout + imageSize = 50 * 1024 * 1024 // 50MB + var digestBytes = Data(count: 32) + digestBytes[0] = 1 + digestBytes[1] = 2 + digestBytes[2] = 3 + imageDigest = try Digest(algorithm: .sha256, bytes: digestBytes) + + case .tarball: + // Simulate loading from tarball + imageSize = 75 * 1024 * 1024 // 75MB + var digestBytes = Data(count: 32) + digestBytes[0] = 4 + digestBytes[1] = 5 + digestBytes[2] = 6 + imageDigest = try Digest(algorithm: .sha256, bytes: digestBytes) + } + + // Create base snapshot + let snapshot = ContainerBuildSnapshotter.Snapshot( + digest: imageDigest, + size: imageSize, + parent: nil as UUID? // Base images have no parent + ) + + // Update context with image config + context.updateImageConfig { config in + // In a real implementation, we'd extract this from the image + config.env = ["PATH=/usr/local/bin:/usr/bin:/bin"] + config.workingDir = "/" + } + + let duration = Date().timeIntervalSince(startTime) + + return ExecutionResult( + filesystemChanges: .empty, + snapshot: snapshot, + duration: duration + ) + } catch { + throw ExecutorError( + type: .executionFailed, + context: ExecutorError.ErrorContext( + operation: operation, underlyingError: error, diagnostics: ExecutorError.Diagnostics(environment: [:], workingDirectory: "", recentLogs: []))) + } + } + + public func canExecute(_ operation: ContainerBuildIR.Operation) -> Bool { + operation is ImageOperation + } +} diff --git a/Sources/NativeBuilder/ContainerBuildExecutor/Executors/MetadataOperationExecutor.swift b/Sources/NativeBuilder/ContainerBuildExecutor/Executors/MetadataOperationExecutor.swift new file mode 100644 index 00000000..6891e982 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildExecutor/Executors/MetadataOperationExecutor.swift @@ -0,0 +1,179 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildSnapshotter +import Foundation + +/// Executes MetadataOperation (ENV, LABEL, USER, etc.). +public struct MetadataOperationExecutor: OperationExecutor { + public let capabilities: ExecutorCapabilities + + public init() { + self.capabilities = ExecutorCapabilities( + supportedOperations: [.metadata], + maxConcurrency: 20 // Metadata ops are lightweight + ) + } + + public func execute(_ operation: ContainerBuildIR.Operation, context: ExecutionContext) async throws -> ExecutionResult { + guard let metadataOp = operation as? MetadataOperation else { + throw ExecutorError( + type: .invalidConfiguration, + context: ExecutorError.ErrorContext( + operation: operation, underlyingError: NSError(domain: "Executor", code: 1), + diagnostics: ExecutorError.Diagnostics(environment: [:], workingDirectory: "", recentLogs: []))) + } + + do { + // Stub implementation + // In a real implementation, this would update the image configuration + + let startTime = Date() + var environmentChanges: [String: EnvironmentValue] = [:] + var metadataChanges: [String: String] = [:] + + // Apply metadata action + switch metadataOp.action { + case .setEnv(let key, let value): + environmentChanges[key] = value + context.updateEnvironment([key: value]) + context.updateImageConfig { $0.env.append("\(key)=\(value)") } + + // Note: There's no unsetEnv in MetadataAction + // This would need to be handled differently in a real implementation + + case .setLabel(let key, let value): + metadataChanges[key] = value + context.updateImageConfig { $0.labels[key] = value } + + case .setUser(let user): + context.setUser(user) + let userString: String + switch user { + case .named(let name): + userString = name + case .uid(let uid): + userString = String(uid) + case .userGroup(let user, let group): + userString = "\(user):\(group)" + case .uidGid(let uid, let gid): + userString = "\(uid):\(gid)" + } + context.updateImageConfig { $0.user = userString } + + case .setWorkdir(let path): + context.setWorkingDirectory(path) + context.updateImageConfig { $0.workingDir = path } + + case .setEntrypoint(let command): + context.updateImageConfig { config in + switch command { + case .exec(let args): + config.entrypoint = args + case .shell(let cmd): + config.entrypoint = ["/bin/sh", "-c", cmd] + } + } + + case .setCmd(let command): + context.updateImageConfig { config in + switch command { + case .exec(let args): + config.cmd = args + case .shell(let cmd): + config.cmd = ["/bin/sh", "-c", cmd] + } + } + + case .expose(let port): + context.updateImageConfig { config in + config.exposedPorts.insert(port.stringValue) + } + + case .setHealthcheck(let healthcheck): + context.updateImageConfig { $0.healthcheck = healthcheck } + + case .setStopSignal(let signal): + context.updateImageConfig { $0.stopSignal = signal } + + case .setShell(let shell): + // Shell affects how commands are executed + metadataChanges["shell"] = shell.joined(separator: " ") + + case .addVolume(let path): + context.updateImageConfig { $0.volumes.insert(path) } + + case .setEnvBatch(let vars): + for (key, value) in vars { + environmentChanges[key] = value + context.updateEnvironment([key: value]) + } + context.updateImageConfig { config in + for (key, value) in vars { + config.env.append("\(key)=\(value)") + } + } + + case .setLabelBatch(let labels): + for (key, value) in labels { + metadataChanges[key] = value + } + context.updateImageConfig { config in + for (key, value) in labels { + config.labels[key] = value + } + } + + case .declareArg(let name, let defaultValue): + // ARG declarations are build-time only + metadataChanges["arg:\(name)"] = defaultValue ?? "" + + case .addOnBuild(let instruction): + // ONBUILD is stored as metadata + metadataChanges["onbuild:\(UUID().uuidString)"] = instruction + } + + // Metadata operations don't change the filesystem + // so we reuse the parent snapshot + let snapshot = + try context.latestSnapshot() + ?? ContainerBuildSnapshotter.Snapshot( + digest: Digest(algorithm: .sha256, bytes: Data(count: 32)), + size: 0 + ) + + let duration = Date().timeIntervalSince(startTime) + + return ExecutionResult( + filesystemChanges: .empty, + environmentChanges: environmentChanges, + metadataChanges: metadataChanges, + snapshot: snapshot, + duration: duration + ) + } catch { + throw ExecutorError( + type: .executionFailed, + context: ExecutorError.ErrorContext( + operation: operation, underlyingError: error, diagnostics: ExecutorError.Diagnostics(environment: [:], workingDirectory: "", recentLogs: []))) + } + } + + public func canExecute(_ operation: ContainerBuildIR.Operation) -> Bool { + operation is MetadataOperation + } +} diff --git a/Sources/NativeBuilder/ContainerBuildExecutor/Executors/UnknownOperationExecutor.swift b/Sources/NativeBuilder/ContainerBuildExecutor/Executors/UnknownOperationExecutor.swift new file mode 100644 index 00000000..369f7584 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildExecutor/Executors/UnknownOperationExecutor.swift @@ -0,0 +1,78 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildSnapshotter +import Foundation + +/// Executes unknown/custom operations using the visitor pattern. +public struct UnknownOperationExecutor: OperationExecutor { + public let capabilities: ExecutorCapabilities + + public init() { + self.capabilities = ExecutorCapabilities( + supportedOperations: [], // Doesn't declare specific operations + maxConcurrency: 1 // Conservative default + ) + } + + public func execute(_ operation: ContainerBuildIR.Operation, context: ExecutionContext) async throws -> ExecutionResult { + do { + // This executor handles operations that don't match built-in types + // In a real implementation, this could: + // 1. Use a plugin system + // 2. Delegate to external executors + // 3. Apply custom logic based on operation metadata + + let startTime = Date() + + // For now, we'll just log and return a no-op result + print("WARNING: Executing unknown operation type: \(type(of: operation))") + print("Operation kind: \(operation.operationKind)") + + // Use the existing snapshot + let snapshot = + try context.latestSnapshot() + ?? ContainerBuildSnapshotter.Snapshot( + digest: Digest(algorithm: .sha256, bytes: Data(count: 32)), + size: 0 + ) + + let duration = Date().timeIntervalSince(startTime) + + return ExecutionResult( + filesystemChanges: .empty, + snapshot: snapshot, + duration: duration, + output: ExecutionOutput( + stdout: "Executed unknown operation: \(operation.operationKind)\n" + ) + ) + } catch { + throw ExecutorError( + type: .executionFailed, + context: ExecutorError.ErrorContext( + operation: operation, underlyingError: error, diagnostics: ExecutorError.Diagnostics(environment: [:], workingDirectory: "", recentLogs: []))) + } + } + + public func canExecute(_ operation: ContainerBuildIR.Operation) -> Bool { + // This executor can handle any operation as a fallback + // In practice, you might want to check for specific metadata + // or operation kinds that indicate custom operations + true + } +} diff --git a/Sources/NativeBuilder/ContainerBuildExecutor/OperationExecutor.swift b/Sources/NativeBuilder/ContainerBuildExecutor/OperationExecutor.swift new file mode 100644 index 00000000..33792fc2 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildExecutor/OperationExecutor.swift @@ -0,0 +1,163 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildSnapshotter +import ContainerizationOCI +import Foundation + +/// Executes individual operations within a build. +/// +/// Implementations handle the actual execution of operations, interacting with +/// the container runtime, filesystem, and other system resources. +public protocol OperationExecutor: Sendable { + /// The capabilities this executor provides. + var capabilities: ExecutorCapabilities { get } + + /// Execute a single operation. + /// + /// - Parameters: + /// - operation: The operation to execute + /// - context: The execution context containing current state + /// - Returns: The result of executing the operation + /// - Throws: Any errors encountered during execution + func execute(_ operation: ContainerBuildIR.Operation, context: ExecutionContext) async throws -> ExecutionResult + + /// Check if this executor can handle the given operation. + /// + /// - Parameter operation: The operation to check + /// - Returns: true if this executor can handle the operation + func canExecute(_ operation: ContainerBuildIR.Operation) -> Bool +} + +/// Describes the capabilities of an executor. +/// +/// Used by the dispatcher to match operations to appropriate executors. +public struct ExecutorCapabilities: Sendable { + /// The operation kinds this executor can handle. + public let supportedOperations: Set + + /// Platform constraints (nil means all platforms). + public let supportedPlatforms: Set? + + /// Whether this executor requires privileged access. + public let requiresPrivileged: Bool + + /// Maximum concurrent operations this executor can handle. + public let maxConcurrency: Int + + /// Resource requirements. + public let resources: ResourceRequirements + + public init( + supportedOperations: Set, + supportedPlatforms: Set? = nil, + requiresPrivileged: Bool = false, + maxConcurrency: Int = 10, + resources: ResourceRequirements = .default + ) { + self.supportedOperations = supportedOperations + self.supportedPlatforms = supportedPlatforms + self.requiresPrivileged = requiresPrivileged + self.maxConcurrency = maxConcurrency + self.resources = resources + } +} + +/// Resource requirements for an executor. +public struct ResourceRequirements: Sendable { + /// Minimum available memory in bytes. + public let minMemory: Int64? + + /// Minimum available disk space in bytes. + public let minDiskSpace: Int64? + + /// Required CPU architecture. + public let cpuArchitecture: String? + + /// Custom requirements. + public let custom: [String: String] + + public init( + minMemory: Int64? = nil, + minDiskSpace: Int64? = nil, + cpuArchitecture: String? = nil, + custom: [String: String] = [:] + ) { + self.minMemory = minMemory + self.minDiskSpace = minDiskSpace + self.cpuArchitecture = cpuArchitecture + self.custom = custom + } + + /// Default resource requirements. + public static let `default` = ResourceRequirements() +} + +/// The result of executing an operation. +public struct ExecutionResult: Sendable { + /// Filesystem changes made by the operation. + public let filesystemChanges: FilesystemChanges + + /// Environment changes made by the operation. + public let environmentChanges: [String: EnvironmentValue] + + /// Metadata changes (labels, etc.). + public let metadataChanges: [String: String] + + /// The snapshot after execution. + public let snapshot: Snapshot + + /// Execution duration. + public let duration: TimeInterval + + /// Any output produced. + public let output: ExecutionOutput? + + public init( + filesystemChanges: FilesystemChanges = .empty, + environmentChanges: [String: EnvironmentValue] = [:], + metadataChanges: [String: String] = [:], + snapshot: Snapshot, + duration: TimeInterval, + output: ExecutionOutput? = nil + ) { + self.filesystemChanges = filesystemChanges + self.environmentChanges = environmentChanges + self.metadataChanges = metadataChanges + self.snapshot = snapshot + self.duration = duration + self.output = output + } +} + +/// Output from operation execution. +public struct ExecutionOutput: Sendable { + /// Standard output. + public let stdout: String + + /// Standard error. + public let stderr: String + + /// Exit code (for exec operations). + public let exitCode: Int? + + public init(stdout: String = "", stderr: String = "", exitCode: Int? = nil) { + self.stdout = stdout + self.stderr = stderr + self.exitCode = exitCode + } +} diff --git a/Sources/NativeBuilder/ContainerBuildExecutor/ReportingHelpers.swift b/Sources/NativeBuilder/ContainerBuildExecutor/ReportingHelpers.swift new file mode 100644 index 00000000..eeadac18 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildExecutor/ReportingHelpers.swift @@ -0,0 +1,202 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildReporting +import Foundation + +// MARK: - Helper Extensions + +extension ReportContext { + /// Create context from a build node and stage + public init(node: BuildNode, stage: BuildStage, operation: any ContainerBuildIR.Operation) { + self.init( + nodeId: node.id, + stageId: stage.name ?? "stage-\(stage.id.uuidString.prefix(8))", + description: Self.describeOperation(operation), + timestamp: Date(), + sourceMap: nil + ) + } + + /// Generate a human-readable description for an operation + public static func describeOperation(_ operation: any ContainerBuildIR.Operation) -> String { + switch operation { + case let exec as ExecOperation: + return "RUN \(exec.command.displayString)" + + case let fs as FilesystemOperation: + switch fs.action { + case .copy: + return "COPY \(Self.describeSource(fs.source)) \(fs.destination)" + case .add: + return "ADD \(Self.describeSource(fs.source)) \(fs.destination)" + case .remove: + return "REMOVE \(fs.destination)" + case .mkdir: + return "MKDIR \(fs.destination)" + case .symlink: + return "SYMLINK \(fs.destination)" + case .hardlink: + return "HARDLINK \(fs.destination)" + } + + case let img as ImageOperation: + switch img.source { + case .registry(let ref): + return "FROM \(ref.stringValue)" + case .scratch: + return "FROM scratch" + case .ociLayout(let path, let tag): + return "FROM oci-layout:\(path)\(tag.map { ":\($0)" } ?? "")" + case .tarball(let path): + return "FROM tarball:\(path)" + } + + case let meta as MetadataOperation: + switch meta.action { + case .setEnv(let key, let value): + return "ENV \(key)=\(Self.describeEnvValue(value))" + case .setEnvBatch(let vars): + return "ENV \(vars.map { "\($0.key)=\(Self.describeEnvValue($0.value))" }.joined(separator: " "))" + case .setWorkdir(let path): + return "WORKDIR \(path)" + case .setUser(let user): + return "USER \(Self.describeUser(user))" + case .setEntrypoint(let cmd): + return "ENTRYPOINT \(cmd.displayString)" + case .setCmd(let cmd): + return "CMD \(cmd.displayString)" + case .setLabel(let key, let value): + return "LABEL \(key)=\(value)" + case .setLabelBatch(let labels): + return "LABEL \(labels.map { "\($0.key)=\($0.value)" }.joined(separator: " "))" + case .declareArg(let name, let defaultValue): + return "ARG \(name)\(defaultValue.map { "=\($0)" } ?? "")" + case .expose(let port): + return "EXPOSE \(port.stringValue)" + case .setStopSignal(let signal): + return "STOPSIGNAL \(signal)" + case .setHealthcheck(let hc): + guard let hc = hc else { + return "HEALTHCHECK NONE" + } + switch hc.test { + case .none: + return "HEALTHCHECK NONE" + case .command(let cmd): + return "HEALTHCHECK CMD \(cmd.displayString)" + case .shell(let cmd): + return "HEALTHCHECK CMD-SHELL \(cmd)" + } + case .setShell(let shell): + return "SHELL [\(shell.map { "\"\($0)\"" }.joined(separator: ", "))]" + case .addVolume(let path): + return "VOLUME \(path)" + case .addOnBuild(let instruction): + return "ONBUILD \(instruction)" + } + + default: + return "Operation \(operation.operationKind.rawValue)" + } + } + + private static func describeSource(_ source: FilesystemSource) -> String { + switch source { + case .context(let ctx): + return "\(ctx.name):\(ctx.paths.joined(separator: " "))" + case .stage(let ref, let paths): + let prefix: String + switch ref { + case .named(let name): + prefix = name + case .index(let idx): + prefix = "stage-\(idx)" + case .previous: + prefix = "previous" + } + return "\(prefix):\(paths.joined(separator: " "))" + case .image(let ref, let paths): + return "\(ref.stringValue):\(paths.joined(separator: " "))" + case .url(let url): + return url.absoluteString + case .git(let src): + return src.repository + case .inline(_): + return "" + case .scratch: + return "scratch" + } + } + + private static func describeEnvValue(_ value: EnvironmentValue) -> String { + switch value { + case .literal(let str): + return str + case .buildArg(let name): + return "${\(name)}" + case .expansion(let name, let defaultValue): + guard let defaultValue = defaultValue else { + return "${\(name)}" + } + return "${\(name):-\(defaultValue)}" + } + } + + private static func describeUser(_ user: User) -> String { + switch user { + case .named(let name): + return name + case .uid(let uid): + return String(uid) + case .userGroup(let user, let group): + return "\(user):\(group)" + case .uidGid(let uid, let gid): + return "\(uid):\(gid)" + } + } +} + +extension BuildEventError { + /// Create from ExecutorError + public init(from executorError: ExecutorError) { + let failureType: FailureType + switch executorError.type { + case .executionFailed: + failureType = .executionFailed + case .cancelled: + failureType = .cancelled + case .invalidConfiguration: + failureType = .invalidConfiguration + } + + var diags: [String: String] = [:] + diags["workingDirectory"] = executorError.context.diagnostics.workingDirectory + for (key, value) in executorError.context.diagnostics.environment { + diags["env.\(key)"] = value + } + if !executorError.context.diagnostics.recentLogs.isEmpty { + diags["recentLogs"] = executorError.context.diagnostics.recentLogs.joined(separator: "\n") + } + + self.init( + type: failureType, + description: executorError.context.underlyingError.localizedDescription, + diagnostics: diags + ) + } +} diff --git a/Sources/NativeBuilder/ContainerBuildExecutor/Scheduler.swift b/Sources/NativeBuilder/ContainerBuildExecutor/Scheduler.swift new file mode 100644 index 00000000..13e04c83 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildExecutor/Scheduler.swift @@ -0,0 +1,1457 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildCache +import ContainerBuildIR +import ContainerBuildReporting +import ContainerBuildSnapshotter +import ContainerizationOCI +import Crypto +import Foundation + +// Import specific type to avoid ambiguity with ContainerBuildIR.CacheKey +import struct ContainerBuildCache.CacheKey +import struct ContainerBuildCache.CacheStatistics +import struct ContainerBuildCache.CachedResult + +/// A production-ready, highly parallel scheduler that minimizes build time through +/// intelligent scheduling and maximum parallelization. +/// +/// Key features: +/// - Parallel execution of independent operations +/// - Dynamic work stealing for load balancing +/// - Resource-aware scheduling with throttling +/// - Priority-based execution ordering +/// - Real-time performance monitoring +/// - Integrated progress reporting via event streams +public final class Scheduler: BuildExecutor { + /// Atomic storage for reporter to maintain Sendable conformance + private let reporterStorage = AtomicStorage() + /// The reporter for this scheduler instance (if progress reporting is enabled) + public var reporter: Reporter? { reporterStorage.value } + /// Completion handler to wait for all consumers + private let completionHandlers = AtomicStorage<[@Sendable () async -> Void]>(initialValue: []) + private let dispatcher: ExecutionDispatcher + private let snapshotter: any Snapshotter + private let cache: any BuildCache + private let configuration: Configuration + + /// Scheduler configuration + public struct Configuration: Sendable { + /// Maximum number of concurrent operations + public let maxConcurrency: Int + + /// Maximum memory usage in bytes + public let maxMemoryUsage: Int64 + + /// Enable work stealing between queues + public let enableWorkStealing: Bool + + /// Enable priority scheduling + public let enablePriorityScheduling: Bool + + /// Resource monitoring interval + public let monitoringInterval: TimeInterval + + /// Fail fast on first error + public let failFast: Bool + + /// Enable progress reporting + public let enableProgressReporting: Bool + + public init( + maxConcurrency: Int = ProcessInfo.processInfo.activeProcessorCount * 2, + maxMemoryUsage: Int64 = 8 * 1024 * 1024 * 1024, // 8GB default + enableWorkStealing: Bool = true, + enablePriorityScheduling: Bool = true, + monitoringInterval: TimeInterval = 0.5, + failFast: Bool = true, + enableProgressReporting: Bool = true + ) { + self.maxConcurrency = maxConcurrency + self.maxMemoryUsage = maxMemoryUsage + self.enableWorkStealing = enableWorkStealing + self.enablePriorityScheduling = enablePriorityScheduling + self.monitoringInterval = monitoringInterval + self.failFast = failFast + self.enableProgressReporting = enableProgressReporting + } + } + + /// Execution state tracking + private let executionState = ExecutionState() + + /// Work queues for parallel execution + private let workQueues: WorkQueueManager + + /// Resource monitor + private let resourceMonitor: ResourceMonitor + + /// Metrics collector + private let metricsCollector = MetricsCollector() + + public init( + executors: [any OperationExecutor]? = nil, + snapshotter: (any Snapshotter)? = nil, + cache: (any BuildCache)? = nil, + reporter: Reporter? = nil, + configuration: Configuration = Configuration() + ) { + let defaultExecutors: [any OperationExecutor] = [ + ImageOperationExecutor(), + ExecOperationExecutor(), + FilesystemOperationExecutor(), + MetadataOperationExecutor(), + UnknownOperationExecutor(), + ] + + self.dispatcher = ExecutionDispatcher(executors: executors ?? defaultExecutors) + self.snapshotter = snapshotter ?? MemorySnapshotter() + self.cache = cache ?? MemoryBuildCache() + self.configuration = configuration + self.workQueues = WorkQueueManager( + concurrency: configuration.maxConcurrency, + enableWorkStealing: configuration.enableWorkStealing + ) + self.resourceMonitor = ResourceMonitor( + maxMemory: configuration.maxMemoryUsage, + interval: configuration.monitoringInterval + ) + + // Initialize reporter based on configuration + if let reporter = reporter { + self.reporterStorage.value = reporter + } else if configuration.enableProgressReporting { + self.reporterStorage.value = Reporter() + } + } + + /// Cancel all in-flight operations and prevent new ones from starting + public func cancel() async { + await executionState.cancel() + await workQueues.cancelAll() + } + + public func execute(_ graph: BuildGraph) async throws -> BuildResult { + let startTime = Date() + + // Reset state + await executionState.reset() + await metricsCollector.reset() + + // Report build started if we have a reporter + if let reporter = reporter { + let totalOperations = graph.stages.reduce(0) { $0 + $1.nodes.count + 1 } // +1 for base image + await reporter.report(.buildStarted(totalOperations: totalOperations, stages: graph.stages.count, timestamp: Date())) + } + + // Start resource monitoring + let monitoringTask = Task { + await resourceMonitor.startMonitoring(executionState: executionState) + } + + defer { + monitoringTask.cancel() + } + + // Analyze graph for parallelization opportunities + let parallelizationPlan = try analyzeGraph(graph) + + // Execute platforms in parallel when possible + let platformResults: [Platform: ImageManifest] + do { + platformResults = try await withThrowingTaskGroup(of: PlatformResult.self) { group in + for platform in graph.targetPlatforms { + group.addTask { + try await self.executePlatform( + graph: graph, + platform: platform, + plan: parallelizationPlan + ) + } + } + + var results: [Platform: ImageManifest] = [:] + do { + for try await result in group { + results[result.platform] = result.manifest + } + } catch { + // Cancel all remaining tasks on error + group.cancelAll() + // Signal cancellation to execution state + await executionState.cancel() + throw error + } + return results + } + } catch { + // Report build failure + await reporter?.report(.buildCompleted(success: false, timestamp: Date())) + await reporter?.finish() + + // Run completion handlers before throwing + for handler in completionHandlers.value { + await handler() + } + + throw error + } + + // Report build success + await reporter?.report(.buildCompleted(success: true, timestamp: Date())) + await reporter?.finish() + + // Run all completion handlers to ensure consumers finish + for handler in completionHandlers.value { + await handler() + } + + // Collect final metrics + let totalDuration = Date().timeIntervalSince(startTime) + let (metrics, logs) = await metricsCollector.finalizeMetrics(totalDuration: totalDuration, executionState: executionState) + let cacheStats = await cache.statistics() + + return BuildResult( + manifests: platformResults, + metrics: metrics, + cacheStats: cacheStats, + logs: logs + ) + } + + /// Register a completion handler that will be called after the build completes + /// but before execute() returns. This is useful for ensuring progress consumers + /// finish processing all events. + public func onCompletion(_ handler: @escaping @Sendable () async -> Void) { + completionHandlers.value.append(handler) + } + + // MARK: - Graph Analysis + + internal func analyzeGraph(_ graph: BuildGraph) throws -> ParallelizationPlan { + var plan = ParallelizationPlan() + + for stage in graph.stages { + // Analyze dependencies within stage + let analysis = try analyzeStage(stage) + plan.stageAnalyses[stage.id] = analysis + } + + return plan + } + + private func analyzeStage(_ stage: BuildStage) throws -> StageAnalysis { + let dependencyGraph = try buildDependencyGraph(stage) + let parallelizableGroups = findParallelizableGroups(dependencyGraph) + + return StageAnalysis( + dependencyGraph: dependencyGraph, + parallelizableGroups: parallelizableGroups + ) + } + + private func buildDependencyGraph(_ stage: BuildStage) throws -> DependencyGraph { + var graph = DependencyGraph() + + // Add all nodes + for node in stage.nodes { + graph.addNode(node) + } + + // Add edges based on dependencies + for node in stage.nodes { + for dep in node.dependencies { + if let depNode = stage.nodes.first(where: { $0.id == dep }) { + graph.addEdge(from: depNode, to: node) + } + } + } + + // Verify no cycles + if graph.hasCycle() { + throw BuildExecutorError.cyclicDependency + } + + return graph + } + + private func findParallelizableGroups(_ graph: DependencyGraph) -> [[BuildNode]] { + var groups: [[BuildNode]] = [] + var processed = Set() + + // Use Kahn's algorithm to find nodes that can execute in parallel + while processed.count < graph.nodeCount { + var currentGroup: [BuildNode] = [] + + // Find all nodes with no unprocessed dependencies + for node in graph.allNodes { + if !processed.contains(node.id) { + let deps = graph.dependencies(of: node) + if deps.allSatisfy({ processed.contains($0.id) }) { + currentGroup.append(node) + } + } + } + + if currentGroup.isEmpty { + break // Shouldn't happen if no cycles + } + + groups.append(currentGroup) + currentGroup.forEach { processed.insert($0.id) } + } + + return groups + } + + // MARK: - Platform Execution + + private func executePlatform( + graph: BuildGraph, + platform: Platform, + plan: ParallelizationPlan + ) async throws -> PlatformResult { + let stages = try graph.stagesForExecution(targetStage: graph.targetStage) + var stageSnapshots: [String: Snapshot] = [:] + var finalSnapshot: Snapshot? + + // Build stage dependency graph to find parallelizable stages + var stageDependencies: [UUID: Set] = [:] + for stage in stages { + stageDependencies[stage.id] = Set() + + // Check for COPY --from dependencies + for node in stage.nodes { + if let copyOp = node.operation as? FilesystemOperation, + case .stage(let stageRef, _) = copyOp.source + { + if let depStage = resolveStageReference(stageRef, in: stages, currentStage: stage) { + stageDependencies[stage.id]?.insert(depStage.id) + } + } + } + } + + // Execute all base images in parallel first + var baseImageSnapshots: [UUID: Snapshot] = [:] + let sharedContext = SharedStageContext() + + // Start all base image operations in parallel + try await withThrowingTaskGroup(of: (UUID, String?, Snapshot?).self) { group in + for stage in stages { + group.addTask { + if await self.executionState.isCancelled { + throw BuildExecutorError.cancelled + } + + let context = ExecutionContext( + stage: stage, + graph: graph, + platform: platform, + reporter: self.reporter ?? Reporter() + ) + + let stageName = stage.name ?? "stage-\(stage.id.uuidString.prefix(8))" + let baseNodeId = UUID() + let baseReportContext = ReportContext( + nodeId: baseNodeId, + stageId: stageName, + description: ReportContext.describeOperation(stage.base), + timestamp: Date(), + sourceMap: nil + ) + + await self.reporter?.report(.operationStarted(context: baseReportContext)) + + let baseSnapshot = try await self.executeBaseImage(stage.base, context: context) + + await self.reporter?.report(.operationFinished(context: baseReportContext, duration: 0)) + + return (stage.id, stage.name, baseSnapshot) + } + } + + // Collect base image results and store in shared context + for try await (stageId, stageName, snapshot) in group { + if let snapshot = snapshot { + baseImageSnapshots[stageId] = snapshot + // Store in shared context so stages can access via COPY --from + if let name = stageName { + await sharedContext.setSnapshot(name, snapshot: snapshot) + } + } + } + } + + // Execute stages in parallel when possible + var completedStages = Set() + var stageResults: [UUID: Snapshot] = [:] + + while completedStages.count < stages.count { + if await executionState.isCancelled { + throw BuildExecutorError.cancelled + } + + // Find stages that can run now (all dependencies completed) + var stagesToRun: [BuildStage] = [] + for stage in stages { + if !completedStages.contains(stage.id) { + let deps = stageDependencies[stage.id] ?? [] + if deps.isSubset(of: completedStages) { + stagesToRun.append(stage) + } + } + } + + if stagesToRun.isEmpty { + throw BuildExecutorError.cyclicDependency + } + + // Execute all ready stages in parallel + try await withThrowingTaskGroup(of: (UUID, String?, Snapshot).self) { group in + for stage in stagesToRun { + let baseSnapshot = baseImageSnapshots[stage.id] + group.addTask { + // Check for cancellation before starting + if await self.executionState.isCancelled { + throw BuildExecutorError.cancelled + } + + let context = ExecutionContext( + stage: stage, + graph: graph, + platform: platform, + reporter: self.reporter ?? Reporter() + ) + + let stageName = stage.name ?? "stage-\(stage.id.uuidString.prefix(8))" + await self.reporter?.report(.stageStarted(stageName: stageName, timestamp: Date())) + + guard let stageAnalysis = plan.stageAnalyses[stage.id] else { + throw BuildExecutorError.internalError("Stage analysis not found for stage \(stage.id)") + } + + // Set the base image snapshot in the context if available + if let baseSnapshot = baseSnapshot { + context.setSnapshot(baseSnapshot, for: UUID()) + await self.executionState.markNodeCompleted(UUID()) + } + + let snapshot = try await self.executeStageParallel( + stage, + context: context, + sharedContext: sharedContext, + plan: stageAnalysis, + skipBaseImage: true + ) + + await self.reporter?.report(.stageCompleted(stageName: stageName, timestamp: Date())) + + return (stage.id, stage.name, snapshot) + } + } + + // Collect results + for try await (stageId, stageName, snapshot) in group { + completedStages.insert(stageId) + stageResults[stageId] = snapshot + if let name = stageName { + await sharedContext.setSnapshot(name, snapshot: snapshot) + stageSnapshots[name] = snapshot + } + finalSnapshot = snapshot + } + } + } + + guard let snapshot = finalSnapshot else { + throw BuildExecutorError.stageNotFound("No stages executed") + } + + let configDigest = try Digest(algorithm: .sha256, bytes: Data(count: 32)) + let manifest = ImageManifest( + digest: snapshot.digest, + size: snapshot.size, + configDigest: configDigest, + layers: [LayerDescriptor(digest: snapshot.digest, size: snapshot.size)] + ) + + return PlatformResult(platform: platform, manifest: manifest) + } + + private func resolveStageReference(_ ref: StageReference, in stages: [BuildStage], currentStage: BuildStage) -> BuildStage? { + switch ref { + case .named(let name): + return stages.first { $0.name == name } + case .index(let idx): + return idx < stages.count ? stages[idx] : nil + case .previous: + if let currentIndex = stages.firstIndex(where: { $0.id == currentStage.id }), currentIndex > 0 { + return stages[currentIndex - 1] + } + return nil + } + } + + private func executeStageParallel( + _ stage: BuildStage, + context: ExecutionContext, + sharedContext: SharedStageContext, + plan: StageAnalysis, + skipBaseImage: Bool = false + ) async throws -> Snapshot { + let stageStart = Date() + defer { + let duration = Date().timeIntervalSince(stageStart) + Task { + await metricsCollector.recordStageDuration(stage.name ?? "unnamed", duration: duration) + } + } + + // Execute base image (unless it was already executed) + if !skipBaseImage { + let baseNodeId = UUID() + let baseReportContext = ReportContext( + nodeId: baseNodeId, + stageId: stage.name ?? "stage-\(stage.id.uuidString.prefix(8))", + description: ReportContext.describeOperation(stage.base), + timestamp: Date(), + sourceMap: nil + ) + + // Report base image operation started + await context.reporter.report(.operationStarted(context: baseReportContext)) + + if let baseSnapshot = try await executeBaseImage(stage.base, context: context) { + context.setSnapshot(baseSnapshot, for: baseNodeId) + + // Report base image operation finished + await context.reporter.report(.operationFinished(context: baseReportContext, duration: 0)) + + // Mark base operation as completed so nodes can depend on it + await executionState.markNodeCompleted(baseNodeId) + } + } + + // Execute nodes in parallel groups + for (groupIndex, group) in plan.parallelizableGroups.enumerated() { + do { + try await executeNodeGroup(group, context: context, stage: stage) + } catch { + // Handle errors based on configuration + if configuration.failFast { + throw BuildExecutorError.operationFailed( + group.first?.operation ?? UnknownOperation(metadata: OperationMetadata()), + underlying: error + ) + } else { + // Log error and continue if possible + print("Warning: Group \(groupIndex) in stage '\(stage.name ?? "unnamed")' failed: \(error)") + // Mark failed nodes to skip dependents + for node in group { + await executionState.markNodeFailed(node.id) + } + } + } + } + + guard let finalSnapshot = context.latestSnapshot() else { + throw BuildExecutorError.stageNotFound("No operations in stage") + } + + return finalSnapshot + } + + private func executeNodeGroup( + _ nodes: [BuildNode], + context: ExecutionContext, + stage: BuildStage + ) async throws { + // Wait for available execution slots + await resourceMonitor.waitForResources(count: nodes.count) + + try await withThrowingTaskGroup(of: Void.self) { group in + for node in nodes { + group.addTask { + try await self.executeNodeWithTracking(node, context: context, stage: stage) + } + } + + do { + try await group.waitForAll() + } catch { + // On error, cancel all remaining tasks + group.cancelAll() + + // Signal cancellation if configured for fail-fast + if self.configuration.failFast { + await self.executionState.cancel() + } + + throw error + } + } + } + + private func executeNodeWithTracking( + _ node: BuildNode, + context: ExecutionContext, + stage: BuildStage + ) async throws { + // Wait for ALL dependencies (including cross-stage) to complete + for depId in node.dependencies { + do { + try await executionState.waitForNode(depId) + } catch { + await executionState.markNodeFailed(node.id) + throw BuildExecutorError.operationFailed( + node.operation, + underlying: error + ) + } + } + + let nodeStart = Date() + await executionState.incrementOperationCount() + + // Create report context + let reportContext = ReportContext(node: node, stage: stage, operation: node.operation) + + // Report operation started + await context.reporter.report(.operationStarted(context: reportContext)) + + defer { + let duration = Date().timeIntervalSince(nodeStart) + Task { + await metricsCollector.recordOperationDuration(node.id, duration: duration) + await resourceMonitor.releaseResource() + } + } + + // Check cache + let cacheKey: CacheKey + do { + cacheKey = try computeCacheKey(node: node, context: context) + } catch { + // If we can't compute cache key, skip caching and execute directly + cacheKey = CacheKey( + operationDigest: try Digest(algorithm: .sha256, bytes: Data(count: 32)), + inputDigests: [], + platform: context.platform + ) + } + + if let cached = await cache.get(cacheKey, for: node.operation) { + await executionState.incrementCacheHits() + context.setSnapshot(cached.snapshot, for: node.id) + + // Report cache hit + await context.reporter.report(.operationCacheHit(context: reportContext)) + + // Mark node as completed + await executionState.markNodeCompleted(node.id) + return + } + + // Execute operation with retry logic + var lastError: Error? + var retryCount = 0 + let retryPolicy = node.operation.metadata.retryPolicy + let maxRetries = retryPolicy.maxRetries + + while retryCount <= maxRetries { + // Check for cancellation before each attempt + if await executionState.isCancelled { + throw BuildExecutorError.cancelled + } + + do { + let result = try await self.executeNode(node, context: context) + + if let output = result.output { + if !output.stdout.isEmpty { + await metricsCollector.recordLog("[\(node.id)] \(output.stdout)") + // Report operation log + await context.reporter.report(.operationLog(context: reportContext, message: output.stdout)) + } + if !output.stderr.isEmpty { + await metricsCollector.recordLog("[\(node.id)] [STDERR] \(output.stderr)") + // Report operation log for stderr + await context.reporter.report(.operationLog(context: reportContext, message: "[STDERR] \(output.stderr)")) + } + } + + context.setSnapshot(result.snapshot, for: node.id) + + // Store in cache + let cachedResult = CachedResult( + snapshot: result.snapshot, + environmentChanges: result.environmentChanges, + metadataChanges: result.metadataChanges + ) + await cache.put(cachedResult, key: cacheKey, for: node.operation) + + // Report operation finished + await context.reporter.report(.operationFinished(context: reportContext, duration: result.duration)) + + // Mark node as completed + await executionState.markNodeCompleted(node.id) + return // Success + + } catch { + lastError = error + retryCount = await executionState.incrementRetryCount(for: node.id) + + if retryCount <= maxRetries { + // Calculate backoff delay + let delay = min( + retryPolicy.initialDelay * pow(retryPolicy.backoffMultiplier, Double(retryCount - 1)), + retryPolicy.maxDelay + ) + + print("Retrying operation \(node.id) after \(delay)s (attempt \(retryCount)/\(maxRetries))") + try await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000)) + } + } + } + + // All retries failed + await executionState.markNodeFailed(node.id) + + // Report operation failed + let finalError = + lastError + ?? BuildExecutorError.operationFailed( + node.operation, + underlying: UnknownFailureError() + ) + let eventError = BuildEventError( + type: .executionFailed, + description: finalError.localizedDescription, + diagnostics: nil + ) + await context.reporter.report(.operationFailed(context: reportContext, error: eventError)) + + throw BuildExecutorError.operationFailed(node.operation, underlying: finalError) + } + + // MARK: - Node Execution + + private func executeBaseImage( + _ operation: ImageOperation, + context: ExecutionContext + ) async throws -> Snapshot? { + let result = try await dispatcher.dispatch(operation, context: context) + return result.snapshot + } + + private func executeNode( + _ node: BuildNode, + context: ExecutionContext + ) async throws -> ExecutionResult { + let constraints = buildNodeConstraints(node) + + return try await dispatcher.dispatch( + node.operation, + context: context, + constraints: constraints + ) + } + + private func buildNodeConstraints(_ node: BuildNode) -> NodeConstraints? { + guard !node.constraints.isEmpty else { return nil } + + var requiresPrivileged = false + var minMemory: Int64? + var cpuArchitecture: String? + + for constraint in node.constraints { + switch constraint { + case .requiresPrivileged: + requiresPrivileged = true + case .memoryLimit(let limit): + minMemory = Int64(limit) + case .requiresPlatform(let platform): + cpuArchitecture = platform.architecture + default: + break + } + } + + return NodeConstraints( + requiresPrivileged: requiresPrivileged, + minMemory: minMemory, + cpuArchitecture: cpuArchitecture + ) + } + + // MARK: - Cache Key Generation + + private func computeCacheKey( + node: BuildNode, + context: ExecutionContext + ) throws -> CacheKey { + let operationDigest = try node.operation.contentDigest() + + var inputDigests: [ContainerBuildIR.Digest] = [] + + // Add parent snapshot digest + if let parentSnapshot = context.latestSnapshot() { + inputDigests.append(parentSnapshot.digest) + } + + // Add dependency snapshots + for depId in node.dependencies { + if let depSnapshot = context.snapshot(for: depId) { + inputDigests.append(depSnapshot.digest) + } + } + + return CacheKey( + operationDigest: operationDigest, + inputDigests: inputDigests.sorted(by: { $0.stringValue < $1.stringValue }), + platform: context.platform + ) + } + +} + +// MARK: - Supporting Types + +/// Shared context for stages running in parallel +private actor SharedStageContext { + private var snapshots: [String: Snapshot] = [:] + + func setSnapshot(_ name: String, snapshot: Snapshot) { + snapshots[name] = snapshot + } + + func getSnapshot(_ name: String) -> Snapshot? { + snapshots[name] + } + + func getAllSnapshots() -> [String: Snapshot] { + snapshots + } +} + +/// Tracks execution state across the scheduler +private actor ExecutionState { + private var cancelled = false + private var operationCount = 0 + private var cacheHits = 0 + private var failedNodes: Set = [] + private var nodeRetries: [UUID: Int] = [:] + private var completedNodes: Set = [] + private var nodeCompletionWaiters: [UUID: [CheckedContinuation]] = [:] + + var isCancelled: Bool { cancelled } + + func cancel() { + cancelled = true + // Wake up any waiters with cancellation error + for (_, waiters) in nodeCompletionWaiters { + for waiter in waiters { + waiter.resume(throwing: BuildExecutorError.cancelled) + } + } + nodeCompletionWaiters.removeAll() + } + + func reset() { + cancelled = false + operationCount = 0 + cacheHits = 0 + failedNodes.removeAll() + nodeRetries.removeAll() + completedNodes.removeAll() + nodeCompletionWaiters.removeAll() + } + + func incrementOperationCount() { + operationCount += 1 + } + + func incrementCacheHits() { + cacheHits += 1 + } + + func markNodeCompleted(_ nodeId: UUID) { + completedNodes.insert(nodeId) + // Wake up any waiters for this node + if let waiters = nodeCompletionWaiters.removeValue(forKey: nodeId) { + for waiter in waiters { + waiter.resume() + } + } + } + + func markNodeFailed(_ nodeId: UUID) { + failedNodes.insert(nodeId) + // Wake up any waiters with failure + if let waiters = nodeCompletionWaiters.removeValue(forKey: nodeId) { + for waiter in waiters { + waiter.resume(throwing: DependencyFailedError(dependencyId: nodeId)) + } + } + } + + func isNodeCompleted(_ nodeId: UUID) -> Bool { + completedNodes.contains(nodeId) + } + + func isNodeFailed(_ nodeId: UUID) -> Bool { + failedNodes.contains(nodeId) + } + + func waitForNode(_ nodeId: UUID) async throws { + if completedNodes.contains(nodeId) { + return // Already completed + } + + if failedNodes.contains(nodeId) { + throw DependencyFailedError(dependencyId: nodeId) + } + + if cancelled { + throw BuildExecutorError.cancelled + } + + // Wait for the node to complete + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + // Check again in case state changed + if completedNodes.contains(nodeId) { + continuation.resume() + return + } + + if failedNodes.contains(nodeId) { + continuation.resume(throwing: DependencyFailedError(dependencyId: nodeId)) + return + } + + if cancelled { + continuation.resume(throwing: BuildExecutorError.cancelled) + return + } + + var waiters = nodeCompletionWaiters[nodeId] ?? [] + waiters.append(continuation) + nodeCompletionWaiters[nodeId] = waiters + } + } + + func incrementRetryCount(for nodeId: UUID) -> Int { + let count = (nodeRetries[nodeId] ?? 0) + 1 + nodeRetries[nodeId] = count + return count + } + + func getStats() -> (operations: Int, hits: Int, failures: Int) { + (operationCount, cacheHits, failedNodes.count) + } +} + +/// Manages work queues for parallel execution with work stealing +private actor WorkQueueManager { + private let queues: [WorkQueue] + private let enableWorkStealing: Bool + private var nextQueueIndex = 0 + + init(concurrency: Int, enableWorkStealing: Bool) { + self.queues = (0.. Void) { + // Round-robin with load balancing + let startIndex = nextQueueIndex + var minLoad = Int.max + var targetQueue = queues[startIndex] + + for i in 0.. Void] = [] + private let lock = NSLock() + private var cancelled = false + private var isProcessing = false + private var stealTargets: [WorkQueue] = [] + private let workAvailable = NSCondition() + + var currentLoad: Int { + lock.withLock { tasks.count } + } + + init(id: Int) { + self.id = id + } + + func setStealTargets(_ targets: [WorkQueue]) { + lock.withLock { + self.stealTargets = targets + } + } + + func enqueue(_ task: @escaping () async throws -> Void) { + lock.withLock { + guard !cancelled else { return } + tasks.append(task) + workAvailable.signal() + } + } + + func processLoop() async { + while !cancelled { + if let task = dequeueOrSteal() { + do { + try await task() + } catch { + // Log error but continue processing + print("Task failed: \(error)") + } + } else { + // No work available, wait + lock.withLock { + guard !cancelled && tasks.isEmpty else { return } + workAvailable.wait() + } + } + } + } + + private func dequeueOrSteal() -> (() async throws -> Void)? { + // Try to get from own queue first + if let task = dequeue() { + return task + } + + // If work stealing is enabled, try to steal from others + if !stealTargets.isEmpty { + // Randomize steal order to avoid contention + let shuffled = stealTargets.shuffled() + for target in shuffled { + if let task = target.steal() { + return task + } + } + } + + return nil + } + + private func dequeue() -> (() async throws -> Void)? { + lock.withLock { + guard !tasks.isEmpty else { return nil } + return tasks.removeFirst() + } + } + + private func steal() -> (() async throws -> Void)? { + lock.withLock { + // Steal from the back to minimize contention + guard tasks.count > 1 else { return nil } + return tasks.removeLast() + } + } + + func cancel() { + lock.withLock { + cancelled = true + tasks.removeAll() + workAvailable.broadcast() + } + } +} + +/// Monitors resource usage +private actor ResourceMonitor { + private let maxMemory: Int64 + private let interval: TimeInterval + private var availableSlots: Int + private var waiters: [CheckedContinuation] = [] + + init(maxMemory: Int64, interval: TimeInterval) { + self.maxMemory = maxMemory + self.interval = interval + self.availableSlots = ProcessInfo.processInfo.activeProcessorCount * 2 + } + + func startMonitoring(executionState: ExecutionState) async { + while await !executionState.isCancelled { + try? await Task.sleep(nanoseconds: UInt64(interval * 1_000_000_000)) + // Monitor memory and CPU usage + // Adjust available slots based on system load + } + } + + func waitForResources(count: Int) async { + while availableSlots < count { + await withCheckedContinuation { continuation in + waiters.append(continuation) + } + } + availableSlots -= count + } + + func releaseResource() { + availableSlots += 1 + if let waiter = waiters.first { + waiters.removeFirst() + waiter.resume() + } + } +} + +/// Collects execution metrics +private actor MetricsCollector { + private var operationDurations: [UUID: TimeInterval] = [:] + private var stageDurations: [String: TimeInterval] = [:] + private var logs: [String] = [] + private var startTime = Date() + + func reset() { + operationDurations.removeAll() + stageDurations.removeAll() + logs.removeAll() + startTime = Date() + } + + func recordOperationDuration(_ id: UUID, duration: TimeInterval) { + operationDurations[id] = duration + } + + func recordStageDuration(_ name: String, duration: TimeInterval) { + stageDurations[name] = duration + } + + func recordLog(_ log: String) { + logs.append(log) + } + + func finalizeMetrics(totalDuration: TimeInterval, executionState: ExecutionState) async -> (ExecutionMetrics, [String]) { + let stats = await executionState.getStats() + let metrics = ExecutionMetrics( + totalDuration: totalDuration, + stageDurations: stageDurations, + operationCount: operationDurations.count, + cachedOperationCount: stats.hits, + bytesTransferred: 0 // TODO: Track this + ) + return (metrics, logs) + } +} + +/// Result for a single platform build +private struct PlatformResult { + let platform: Platform + let manifest: ImageManifest +} + +/// Parallelization analysis results +internal struct ParallelizationPlan { + var stageAnalyses: [UUID: StageAnalysis] = [:] +} + +/// Analysis results for a single stage +internal struct StageAnalysis { + let dependencyGraph: DependencyGraph + let parallelizableGroups: [[BuildNode]] +} + +/// Dependency graph for analysis +internal struct DependencyGraph { + private var adjacencyList: [UUID: Set] = [:] + private var nodesList: [BuildNode] = [] + + var nodeCount: Int { nodesList.count } + var allNodes: [BuildNode] { nodesList } + + mutating func addNode(_ node: BuildNode) { + nodesList.append(node) + adjacencyList[node.id] = [] + } + + mutating func addEdge(from: BuildNode, to: BuildNode) { + adjacencyList[from.id]?.insert(to.id) + } + + func dependencies(of node: BuildNode) -> [BuildNode] { + nodesList.filter { node.dependencies.contains($0.id) } + } + + func hasCycle() -> Bool { + var visited = Set() + var recursionStack = Set() + + func dfs(_ nodeId: UUID) -> Bool { + visited.insert(nodeId) + recursionStack.insert(nodeId) + + if let neighbors = adjacencyList[nodeId] { + for neighbor in neighbors { + if !visited.contains(neighbor) { + if dfs(neighbor) { + return true + } + } else if recursionStack.contains(neighbor) { + return true + } + } + } + + recursionStack.remove(nodeId) + return false + } + + for node in nodesList { + if !visited.contains(node.id) { + if dfs(node.id) { + return true + } + } + } + + return false + } +} + +// MARK: - Error Types + +/// Error when a dependency failed +struct DependencyFailedError: LocalizedError { + let dependencyId: UUID + + var errorDescription: String? { + "Dependency \(dependencyId) failed" + } +} + +/// Error when operation fails for unknown reasons +struct UnknownFailureError: LocalizedError { + var errorDescription: String? { + "Operation failed for unknown reasons" + } +} + +/// Placeholder for unknown operations +struct UnknownOperation: ContainerBuildIR.Operation { + var metadata: OperationMetadata + static let operationKind = OperationKind(rawValue: "unknown") + var operationKind: OperationKind { Self.operationKind } + + func accept(_ visitor: V) throws -> V.Result where V: OperationVisitor { + throw BuildExecutorError.unsupportedOperation(self) + } +} + +// MARK: - Extensions + +extension BuildGraph { + /// Get stages in execution order for a target stage, resolving all dependencies. + func stagesForExecution(targetStage: BuildStage?) throws -> [BuildStage] { + let target = targetStage ?? stages.last + guard let target = target else { + return [] + } + + // Build dependency graph for stages + var stageDependencies: [UUID: Set] = [:] + var stagesByName: [String: BuildStage] = [:] + var stagesByID: [UUID: BuildStage] = [:] + + // Index stages + for stage in stages { + stagesByID[stage.id] = stage + if let name = stage.name { + stagesByName[name] = stage + } + stageDependencies[stage.id] = [] + } + + // Resolve FROM dependencies + // Note: In the current IR, stage dependencies are handled differently + // The base is always an ImageOperation, not a stage reference + // Stage-to-stage dependencies are handled through COPY --from operations + + // Resolve COPY --from dependencies + for stage in stages { + for node in stage.nodes { + if let copyOp = node.operation as? FilesystemOperation, + case .stage(let stageRef, _) = copyOp.source + { + let stageName: String + switch stageRef { + case .named(let name): + stageName = name + case .index(let idx): + // Find stage by index + guard idx < stages.count else { + throw BuildExecutorError.stageNotFound("Stage index \(idx) out of bounds") + } + stageName = stages[idx].name ?? "stage-\(idx)" + case .previous: + // Find the previous stage + guard let currentIndex = stages.firstIndex(where: { $0.id == stage.id }), + currentIndex > 0 + else { + throw BuildExecutorError.stageNotFound("No previous stage available") + } + stageName = stages[currentIndex - 1].name ?? "stage-\(currentIndex - 1)" + } + + guard let sourceStage = stagesByName[stageName] else { + throw BuildExecutorError.stageNotFound("Stage '\(stageName)' referenced in COPY --from not found") + } + stageDependencies[stage.id]?.insert(sourceStage.id.uuidString) + } + } + } + + // Topological sort to find execution order + var visited = Set() + var recursionStack = Set() + var executionOrder: [BuildStage] = [] + + func visit(_ stageId: UUID) throws { + if recursionStack.contains(stageId) { + throw BuildExecutorError.cyclicDependency + } + + if visited.contains(stageId) { + return + } + + recursionStack.insert(stageId) + + // Visit dependencies first + if let deps = stageDependencies[stageId] { + for depIdString in deps { + if let depId = UUID(uuidString: depIdString), + let _ = stagesByID[depId] + { + try visit(depId) + } + } + } + + recursionStack.remove(stageId) + visited.insert(stageId) + + if let stage = stagesByID[stageId] { + executionOrder.append(stage) + } + } + + // Start from target and work backwards + try visit(target.id) + + // Also visit any stages that the target transitively depends on + var targetDependencies = Set() + func collectDependencies(_ stageId: UUID) { + if let deps = stageDependencies[stageId] { + for depIdString in deps { + if let depId = UUID(uuidString: depIdString) { + if !targetDependencies.contains(depId) { + targetDependencies.insert(depId) + collectDependencies(depId) + } + } + } + } + } + collectDependencies(target.id) + + // Include all required stages + for stage in stages { + if targetDependencies.contains(stage.id) || stage.id == target.id { + if !visited.contains(stage.id) { + try visit(stage.id) + } + } + } + + return executionOrder + } +} + +// MARK: - Thread-Safe Storage + +/// Thread-safe storage for reference types +private final class AtomicStorage: @unchecked Sendable { + private var _value: T + private let lock = NSLock() + + var value: T { + get { + lock.withLock { _value } + } + set { + lock.withLock { _value = newValue } + } + } + + init(initialValue: T) { + self._value = initialValue + } +} + +extension AtomicStorage { + convenience init() where T: ExpressibleByNilLiteral { + self.init(initialValue: nil) + } +} diff --git a/Sources/NativeBuilder/ContainerBuildIR/Analysis/Analyzer.swift b/Sources/NativeBuilder/ContainerBuildIR/Analysis/Analyzer.swift new file mode 100644 index 00000000..0afb7d02 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/Analysis/Analyzer.swift @@ -0,0 +1,44 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildReporting +import Foundation + +/// Context provided to analyzers during analysis +public struct AnalysisContext: Sendable { + /// Reporter for emitting warnings, errors, and other events + public let reporter: Reporter? + + /// Source location information if available + public let sourceMap: SourceMap? + + public init(reporter: Reporter? = nil, sourceMap: SourceMap? = nil) { + self.reporter = reporter + self.sourceMap = sourceMap + } +} + +/// Stage-level analyzer that can transform stages +public protocol StageAnalyzer { + /// Analyze and potentially transform a single stage + func analyze(_ stage: BuildStage, context: AnalysisContext) throws -> BuildStage +} + +/// Graph-level analyzer that can transform the entire build graph +public protocol GraphAnalyzer { + /// Analyze and potentially transform the entire build graph + func analyze(_ graph: BuildGraph, context: AnalysisContext) throws -> BuildGraph +} diff --git a/Sources/NativeBuilder/ContainerBuildIR/Analysis/DependencyAnalyzer.swift b/Sources/NativeBuilder/ContainerBuildIR/Analysis/DependencyAnalyzer.swift new file mode 100644 index 00000000..f4363334 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/Analysis/DependencyAnalyzer.swift @@ -0,0 +1,111 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildReporting +import Foundation + +/// Establishes dependencies between operations in the build graph. +/// +/// This analyzer is responsible for: +/// - Setting up intra-stage dependencies (sequential operations within a stage) +/// - Establishing cross-stage dependencies (COPY --from operations) +/// - Ensuring operations have proper dependencies for correct execution order +public struct DependencyAnalyzer: GraphAnalyzer { + + public init() {} + + public func analyze(_ graph: BuildGraph, context: AnalysisContext) throws -> BuildGraph { + var updatedStages: [BuildStage] = [] + + // Process each stage + for stage in graph.stages { + let updatedStage = try analyzeStage(stage, allStages: graph.stages) + updatedStages.append(updatedStage) + } + + // Create updated graph with new dependencies + return try BuildGraph( + stages: updatedStages, + buildArgs: graph.buildArgs, + targetPlatforms: graph.targetPlatforms, + metadata: graph.metadata + ) + } + + private func analyzeStage(_ stage: BuildStage, allStages: [BuildStage]) throws -> BuildStage { + var updatedNodes: [BuildNode] = [] + var lastNodeId: UUID? = nil + + // Process each node in the stage + for (index, node) in stage.nodes.enumerated() { + var dependencies = node.dependencies + + // 1. Sequential dependency: each operation depends on the previous one + if dependencies.isEmpty && index > 0 { + if let lastId = lastNodeId { + dependencies.insert(lastId) + } + } + + // 2. Cross-stage dependencies: COPY --from operations + if let copyOp = node.operation as? FilesystemOperation { + if case .stage(let stageRef, _) = copyOp.source { + // Find the referenced stage + if let sourceStage = resolveStageReference(stageRef, currentStage: stage, allStages: allStages) { + // COPY --from depends on all operations in the source stage + if let lastOp = sourceStage.nodes.last { + dependencies.insert(lastOp.id) + } + } + } + } + + // Create updated node with dependencies + let updatedNode = BuildNode( + id: node.id, + operation: node.operation, + dependencies: dependencies + ) + + updatedNodes.append(updatedNode) + lastNodeId = updatedNode.id + } + + // Return updated stage + return BuildStage( + id: stage.id, + name: stage.name, + base: stage.base, + nodes: updatedNodes, + platform: stage.platform + ) + } + + private func resolveStageReference(_ ref: StageReference, currentStage: BuildStage, allStages: [BuildStage]) -> BuildStage? { + switch ref { + case .named(let name): + return allStages.first { $0.name == name } + case .index(let idx): + return idx < allStages.count ? allStages[idx] : nil + case .previous: + // Find current stage index + if let currentIndex = allStages.firstIndex(where: { $0.id == currentStage.id }), currentIndex > 0 { + return allStages[currentIndex - 1] + } + return nil + } + } +} diff --git a/Sources/NativeBuilder/ContainerBuildIR/Analysis/SemanticAnalyzer.swift b/Sources/NativeBuilder/ContainerBuildIR/Analysis/SemanticAnalyzer.swift new file mode 100644 index 00000000..9a29d0e3 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/Analysis/SemanticAnalyzer.swift @@ -0,0 +1,482 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildReporting +import Foundation + +/// Performs semantic analysis on build graphs. +/// +/// Design rationale: +/// - Goes beyond structural validation to understand intent +/// - Provides optimization suggestions +/// - Detects common patterns and anti-patterns +public struct SemanticAnalyzer: GraphAnalyzer { + + /// Initialize a new semantic analyzer. + public init() {} + + /// Analyze a build graph and report issues via the reporter + public func analyze(_ graph: BuildGraph, context: AnalysisContext) throws -> BuildGraph { + // Perform the analysis + let analysis = performAnalysis(graph) + + // Report findings via the reporter + reportFindings(analysis, context: context) + + // Return the graph unchanged (semantic analyzer doesn't transform) + return graph + } + + /// Perform semantic analysis on the graph + private func performAnalysis(_ graph: BuildGraph) -> SemanticAnalysis { + let layerAnalysis = analyzeLayerEfficiency(graph) + let cacheAnalysis = analyzeCacheability(graph) + let securityAnalysis = analyzeSecurityPosture(graph) + let sizeAnalysis = analyzeSizeOptimizations(graph) + + return SemanticAnalysis( + layerEfficiency: layerAnalysis, + cacheability: cacheAnalysis, + security: securityAnalysis, + sizeOptimizations: sizeAnalysis + ) + } + + // MARK: - Layer Efficiency + + private func analyzeLayerEfficiency(_ graph: BuildGraph) -> LayerEfficiencyAnalysis { + var issues: [LayerIssue] = [] + + for stage in graph.stages { + // Check for operations that could be combined + let _ = stage.nodes.compactMap { $0.operation as? ExecOperation } + + // Detect multiple package manager invocations + var packageManagerCalls: [(command: String, node: BuildNode)] = [] + + for (_, node) in stage.nodes.enumerated() { + if let execOp = node.operation as? ExecOperation, + case .shell(let cmd) = execOp.command + { + if cmd.contains("apt-get") || cmd.contains("yum") || cmd.contains("apk") || cmd.contains("dnf") { + packageManagerCalls.append((cmd, node)) + } + } + } + + if packageManagerCalls.count > 1 { + issues.append( + LayerIssue( + type: .multipleLayers, + description: "Multiple package manager invocations create separate layers", + suggestion: "Combine package installations into a single RUN command", + estimatedImpact: .high + )) + } + + // Check for file operations followed by deletions + for (index, node) in stage.nodes.enumerated() { + if let fsOp = node.operation as? FilesystemOperation, + fsOp.action == .copy || fsOp.action == .add + { + // Look for subsequent removal + for nextNode in stage.nodes[(index + 1)...] { + if let nextFs = nextNode.operation as? FilesystemOperation, + nextFs.action == .remove + { + issues.append( + LayerIssue( + type: .unnecessaryFiles, + description: "Files added then removed still consume layer space", + suggestion: "Avoid adding files that will be deleted", + estimatedImpact: .medium + )) + } + } + } + } + } + + return LayerEfficiencyAnalysis(issues: issues) + } + + // MARK: - Cache Analysis + + private func analyzeCacheability(_ graph: BuildGraph) -> CacheabilityAnalysis { + var invalidators: [CacheInvalidator] = [] + + for stage in graph.stages { + // Check for operations that frequently invalidate cache + for (index, node) in stage.nodes.enumerated() { + if let fsOp = node.operation as? FilesystemOperation { + switch fsOp.source { + case .context(let source): + // Copying entire context early invalidates cache + if source.paths.contains(".") && index < stage.nodes.count / 2 { + invalidators.append( + CacheInvalidator( + operation: "COPY . .", + reason: "Copying entire context early in build", + suggestion: "Copy only necessary files or move COPY . . later" + )) + } + default: + break + } + } + + // Dynamic commands that change frequently + if let execOp = node.operation as? ExecOperation, + case .shell(let cmd) = execOp.command + { + if cmd.contains("date") || cmd.contains("timestamp") || cmd.contains("git rev-parse") { + invalidators.append( + CacheInvalidator( + operation: cmd, + reason: "Command output changes frequently", + suggestion: "Use build args for dynamic values" + )) + } + } + } + } + + return CacheabilityAnalysis( + cacheInvalidators: invalidators, + estimatedCacheHitRate: invalidators.isEmpty ? 0.8 : 0.3 + ) + } + + // MARK: - Security Analysis + + private func analyzeSecurityPosture(_ graph: BuildGraph) -> SecurityAnalysis { + var findings: [SecurityFinding] = [] + + for stage in graph.stages { + var currentUser: User? + var hasUserSwitch = false + + for node in stage.nodes { + // Track user context + if let metaOp = node.operation as? MetadataOperation, + case .setUser(let user) = metaOp.action + { + currentUser = user + hasUserSwitch = true + } + + // Check for security issues in exec operations + if let execOp = node.operation as? ExecOperation { + // Running privileged without user switch + if execOp.security.privileged && !hasUserSwitch { + findings.append( + SecurityFinding( + severity: .high, + type: .privilegedExecution, + description: "Privileged execution as root", + remediation: "Switch to non-root user after privileged operations" + )) + } + + // Downloading without verification + if case .shell(let cmd) = execOp.command { + if (cmd.contains("curl") || cmd.contains("wget")) && !cmd.contains("--verify") && !cmd.contains("sha256") { + findings.append( + SecurityFinding( + severity: .medium, + type: .unverifiedDownload, + description: "Downloading files without verification", + remediation: "Add checksum verification for downloaded files" + )) + } + + // Installing packages without pinning versions + if cmd.contains("install") && !cmd.contains("=") && (cmd.contains("apt-get") || cmd.contains("pip")) { + findings.append( + SecurityFinding( + severity: .low, + type: .unpinnedDependencies, + description: "Installing packages without version pinning", + remediation: "Pin package versions for reproducible builds" + )) + } + } + } + } + + // Final user check + if currentUser == nil && stage == graph.targetStage { + findings.append( + SecurityFinding( + severity: .high, + type: .rootUser, + description: "Container runs as root by default", + remediation: "Add USER instruction to run as non-root" + )) + } + } + + return SecurityAnalysis(findings: findings) + } + + // MARK: - Size Optimization + + private func analyzeSizeOptimizations(_ graph: BuildGraph) -> SizeOptimizationAnalysis { + var opportunities: [SizeOptimization] = [] + + for stage in graph.stages { + var hasCleanup = false + + for node in stage.nodes { + if let execOp = node.operation as? ExecOperation, + case .shell(let cmd) = execOp.command + { + // Check for package manager cleanup + if cmd.contains("apt-get install") && !cmd.contains("rm -rf /var/lib/apt/lists/*") { + opportunities.append( + SizeOptimization( + type: .packageManagerCache, + description: "Package manager cache not cleaned", + estimatedSavingMB: 50, + suggestion: "Add && rm -rf /var/lib/apt/lists/* after apt-get install" + )) + } + + // Check for build dependencies + if cmd.contains("build-essential") || cmd.contains("-dev") { + let isMultiStage = graph.stages.count > 1 + if !isMultiStage { + opportunities.append( + SizeOptimization( + type: .buildDependencies, + description: "Build dependencies included in final image", + estimatedSavingMB: 200, + suggestion: "Use multi-stage build to exclude build dependencies" + )) + } + } + + if cmd.contains("rm") || cmd.contains("clean") { + hasCleanup = true + } + } + } + + // Check for cleanup in separate layers + if hasCleanup { + opportunities.append( + SizeOptimization( + type: .separateCleanupLayer, + description: "Cleanup in separate RUN creates new layer", + estimatedSavingMB: 0, + suggestion: "Combine cleanup with installation in same RUN" + )) + } + } + + return SizeOptimizationAnalysis(opportunities: opportunities) + } +} + +// MARK: - Analysis Results + +/// Complete semantic analysis results. +public struct SemanticAnalysis { + public let layerEfficiency: LayerEfficiencyAnalysis + public let cacheability: CacheabilityAnalysis + public let security: SecurityAnalysis + public let sizeOptimizations: SizeOptimizationAnalysis + + /// Overall health score (0-100) + public var healthScore: Int { + var score = 100 + + // Deduct for layer issues + score -= layerEfficiency.issues.count * 5 + + // Deduct for cache invalidators + score -= cacheability.cacheInvalidators.count * 10 + + // Deduct for security findings + for finding in security.findings { + switch finding.severity { + case .high: score -= 20 + case .medium: score -= 10 + case .low: score -= 5 + } + } + + // Deduct for size opportunities + score -= min(sizeOptimizations.totalPotentialSavingMB / 50, 20) + + return max(0, score) + } +} + +/// Layer efficiency analysis. +public struct LayerEfficiencyAnalysis { + public let issues: [LayerIssue] +} + +public struct LayerIssue { + public enum IssueType { + case multipleLayers + case unnecessaryFiles + case largeLayer + case inefficientOrdering + } + + public let type: IssueType + public let description: String + public let suggestion: String + public let estimatedImpact: Impact +} + +public enum Impact { + case low, medium, high +} + +/// Cache analysis results. +public struct CacheabilityAnalysis { + public let cacheInvalidators: [CacheInvalidator] + public let estimatedCacheHitRate: Double +} + +public struct CacheInvalidator { + public let operation: String + public let reason: String + public let suggestion: String +} + +/// Security analysis results. +public struct SecurityAnalysis { + public let findings: [SecurityFinding] +} + +public struct SecurityFinding { + public enum Severity { + case low, medium, high + } + + public enum FindingType { + case rootUser + case privilegedExecution + case unverifiedDownload + case unpinnedDependencies + case exposedSecrets + } + + public let severity: Severity + public let type: FindingType + public let description: String + public let remediation: String +} + +/// Size optimization analysis. +public struct SizeOptimizationAnalysis { + public let opportunities: [SizeOptimization] + + public var totalPotentialSavingMB: Int { + opportunities.reduce(0) { $0 + $1.estimatedSavingMB } + } +} + +public struct SizeOptimization { + public enum OptimizationType { + case packageManagerCache + case buildDependencies + case unnecessaryFiles + case separateCleanupLayer + case duplicateFiles + } + + public let type: OptimizationType + public let description: String + public let estimatedSavingMB: Int + public let suggestion: String +} + +// MARK: - Reporting + +extension SemanticAnalyzer { + private func reportFindings(_ analysis: SemanticAnalysis, context: AnalysisContext) { + guard let reporter = context.reporter else { return } + + // Report layer efficiency issues + for issue in analysis.layerEfficiency.issues { + let description = "\(issue.description). \(issue.suggestion)" + let eventType: IREventType = issue.estimatedImpact == .high ? .error : .warning + Task { + await reporter.report( + .irEvent( + context: ReportContext( + description: description, + sourceMap: nil + ), + type: eventType + )) + } + } + + // Report security findings + for finding in analysis.security.findings { + let description = "\(finding.description). \(finding.remediation)" + let eventType: IREventType = finding.severity == .high ? .error : .warning + Task { + await reporter.report( + .irEvent( + context: ReportContext( + description: description, + sourceMap: nil + ), + type: eventType + )) + } + } + + // Report cache invalidators + for invalidator in analysis.cacheability.cacheInvalidators { + let description = "Cache invalidator: \(invalidator.reason). \(invalidator.suggestion)" + Task { + await reporter.report( + .irEvent( + context: ReportContext( + description: description, + sourceMap: nil + ), + type: .warning + )) + } + } + + // Report size optimizations + for optimization in analysis.sizeOptimizations.opportunities { + if optimization.estimatedSavingMB > 100 { + let description = "\(optimization.description). \(optimization.suggestion) (potential saving: \(optimization.estimatedSavingMB)MB)" + Task { + await reporter.report( + .irEvent( + context: ReportContext( + description: description, + sourceMap: nil + ), + type: .warning + )) + } + } + } + } +} diff --git a/Sources/NativeBuilder/ContainerBuildIR/Analysis/ValidatorAdapter.swift b/Sources/NativeBuilder/ContainerBuildIR/Analysis/ValidatorAdapter.swift new file mode 100644 index 00000000..3ba59b6b --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/Analysis/ValidatorAdapter.swift @@ -0,0 +1,76 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildReporting +import Foundation + +/// Adapts a BuildValidator to work as a GraphAnalyzer +public struct ValidatorAnalyzer: GraphAnalyzer { + private let validator: any BuildValidator & Sendable + + public init(validator: V) { + self.validator = validator + } + + public func analyze(_ graph: BuildGraph, context: AnalysisContext) throws -> BuildGraph { + let result = validator.validate(graph) + + // Report errors via reporter + if let reporter = context.reporter { + let sourceMap = context.sourceMap + for error in result.errors { + let description = error.localizedDescription + Task { + await reporter.report( + .irEvent( + context: ReportContext( + description: description, + sourceMap: sourceMap + ), + type: .error + )) + } + } + } + + // Report warnings via reporter + if let reporter = context.reporter { + let sourceMap = context.sourceMap + for warning in result.warnings { + let message = warning.suggestion != nil ? "\(warning.message). \(warning.suggestion!)" : warning.message + + Task { + await reporter.report( + .irEvent( + context: ReportContext( + description: message, + sourceMap: sourceMap + ), + type: .warning + )) + } + } + } + + // Always fail on error as requested + if !result.errors.isEmpty { + throw result.errors.first! + } + + // Validation doesn't transform the graph + return graph + } +} diff --git a/Sources/NativeBuilder/ContainerBuildIR/BuildDefinitionError.swift b/Sources/NativeBuilder/ContainerBuildIR/BuildDefinitionError.swift new file mode 100644 index 00000000..04490dc5 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/BuildDefinitionError.swift @@ -0,0 +1,47 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// An error that occurs during the parsing or validation of a build definition. +public enum BuildDefinitionError: Error, LocalizedError { + /// A syntax error was found at a specific location. + case invalidSyntax(line: Int, column: Int, message: String) + + /// An instruction or command is not recognized. + case unknownInstruction(name: String, line: Int) + + /// An argument for an instruction is invalid. + case invalidArgument(instruction: String, argument: String, reason: String) + + /// A required resource, like a file for a `COPY` command, was not found. + case sourceNotFound(path: String, instructionLine: Int) + + // MARK: - LocalizedError Conformance + + public var errorDescription: String? { + switch self { + case .invalidSyntax(let line, let col, let msg): + return "Syntax error on line \(line):\(col): \(msg)" + case .unknownInstruction(let name, let line): + return "Unknown instruction '\(name)' on line \(line)" + case .invalidArgument(let instruction, let argument, let reason): + return "Invalid argument '\(argument)' for instruction '\(instruction)': \(reason)" + case .sourceNotFound(let path, let line): + return "Source path '\(path)' not found for instruction on line \(line)" + } + } +} diff --git a/Sources/NativeBuilder/ContainerBuildIR/Digest.swift b/Sources/NativeBuilder/ContainerBuildIR/Digest.swift new file mode 100644 index 00000000..4ca3cc4e --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/Digest.swift @@ -0,0 +1,173 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Crypto +import Foundation + +/// A content-addressed identifier for build artifacts. +/// +/// Design rationale: +/// - Immutable value type ensures digests cannot be modified after creation +/// - Strong typing prevents mixing different hash algorithms +/// - Validates format on creation to catch errors early +/// - Supports common container ecosystem digest formats +public struct Digest: Hashable, Sendable { + /// The algorithm used to compute this digest + public enum Algorithm: String, CaseIterable, Sendable { + case sha256 + case sha384 + case sha512 + + /// Expected byte length for this algorithm + var byteLength: Int { + switch self { + case .sha256: return 32 + case .sha384: return 48 + case .sha512: return 64 + } + } + } + + public let algorithm: Algorithm + public let bytes: Data + + /// Create a digest from raw bytes + /// - Throws: If bytes length doesn't match algorithm requirements + public init(algorithm: Algorithm, bytes: Data) throws { + guard bytes.count == algorithm.byteLength else { + throw DigestError.invalidLength(expected: algorithm.byteLength, actual: bytes.count) + } + self.algorithm = algorithm + self.bytes = bytes + } + + /// Create a digest from a hex string (e.g., "sha256:abc123...") + public init(parsing string: String) throws { + let components = string.split(separator: ":", maxSplits: 1) + guard components.count == 2 else { + throw DigestError.invalidFormat(string) + } + + guard let algorithm = Algorithm(rawValue: String(components[0])) else { + throw DigestError.unsupportedAlgorithm(String(components[0])) + } + + guard let bytes = Data(hexString: String(components[1])) else { + throw DigestError.invalidHex(String(components[1])) + } + + try self.init(algorithm: algorithm, bytes: bytes) + } + + /// String representation in standard format (e.g., "sha256:abc123...") + public var stringValue: String { + "\(algorithm.rawValue):\(bytes.hexString)" + } + + /// Compute digest of data + /// - Throws: DigestError.cryptoInternalError if Crypto produces unexpected results + public static func compute(_ data: Data, using algorithm: Algorithm = .sha256) throws -> Digest { + let bytes: Data + switch algorithm { + case .sha256: + var hasher = SHA256() + hasher.update(data: data) + bytes = Data(hasher.finalize()) + case .sha384: + var hasher = SHA384() + hasher.update(data: data) + bytes = Data(hasher.finalize()) + case .sha512: + var hasher = SHA512() + hasher.update(data: data) + bytes = Data(hasher.finalize()) + } + + // This should never fail as Crypto produces the correct byte length + // But we handle it gracefully for production safety + do { + return try Digest(algorithm: algorithm, bytes: bytes) + } catch { + // This should never happen in practice, but we provide proper error handling + throw DigestError.cryptoInternalError("Crypto produced digest with incorrect length for \(algorithm): \(error)") + } + } +} + +extension Digest: CustomStringConvertible { + public var description: String { stringValue } +} + +extension Digest: Codable { + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + let string = try container.decode(String.self) + try self.init(parsing: string) + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(stringValue) + } +} + +public enum DigestError: LocalizedError { + case invalidFormat(String) + case unsupportedAlgorithm(String) + case invalidHex(String) + case invalidLength(expected: Int, actual: Int) + case cryptoInternalError(String) + + public var errorDescription: String? { + switch self { + case .invalidFormat(let string): + return "Invalid digest format: '\(string)'. Expected 'algorithm:hex'." + case .unsupportedAlgorithm(let algo): + return "Unsupported digest algorithm: '\(algo)'" + case .invalidHex(let hex): + return "Invalid hex string: '\(hex)'" + case .invalidLength(let expected, let actual): + return "Invalid digest length: expected \(expected) bytes, got \(actual)" + case .cryptoInternalError(let details): + return "Crypto internal error: \(details)" + } + } +} + +// MARK: - Utility Extensions + +extension Data { + fileprivate init?(hexString: String) { + guard hexString.count % 2 == 0 else { return nil } + + var data = Data(capacity: hexString.count / 2) + + for i in stride(from: 0, to: hexString.count, by: 2) { + let startIndex = hexString.index(hexString.startIndex, offsetBy: i) + let endIndex = hexString.index(startIndex, offsetBy: 2) + let hexByte = hexString[startIndex.. + + /// Graph metadata + public let metadata: BuildGraphMetadata + + public init( + stages: [BuildStage], + buildArgs: [String: String] = [:], + targetPlatforms: Set = [], + metadata: BuildGraphMetadata = BuildGraphMetadata() + ) throws { + self.stages = stages + self.buildArgs = buildArgs + self.targetPlatforms = targetPlatforms.isEmpty ? [Platform.current] : targetPlatforms + self.metadata = metadata + + // Validate graph structure + try Self.validate(stages: stages) + } + + /// Get stage by name + public func stage(named name: String) -> BuildStage? { + stages.first { $0.name == name } + } + + /// Get stage by index + public func stage(at index: Int) -> BuildStage? { + guard index >= 0 && index < stages.count else { return nil } + return stages[index] + } + + /// Resolve a stage reference + public func resolveStage(_ reference: StageReference) -> BuildStage? { + switch reference { + case .named(let name): + return stage(named: name) + case .index(let idx): + return stage(at: idx) + case .previous: + // This needs context of current stage to resolve + return nil + } + } + + /// Get the final stage (build target) + public var targetStage: BuildStage? { + stages.last + } + + // MARK: - Validation + + private static func validate(stages: [BuildStage]) throws { + // Check for duplicate stage names + var seenNames = Set() + for stage in stages { + if let name = stage.name { + guard seenNames.insert(name).inserted else { + throw BuildGraphError.duplicateStageName(name) + } + } + } + + // Collect all node IDs across all stages + var allNodeIds = Set() + for stage in stages { + for node in stage.nodes { + allNodeIds.insert(node.id) + } + } + + // Validate node dependencies (can be cross-stage) + for stage in stages { + for node in stage.nodes { + for dep in node.dependencies { + guard allNodeIds.contains(dep) else { + throw BuildGraphError.invalidDependency(node.id, dep) + } + } + } + } + + // Validate DAG structure (check for cycles across entire graph) + try validateGlobalDAG(stages: stages) + } + + private static func validateGlobalDAG(stages: [BuildStage]) throws { + // Build adjacency list for all nodes across all stages + var adjacencyList: [UUID: Set] = [:] + var allNodes = Set() + + for stage in stages { + for node in stage.nodes { + allNodes.insert(node.id) + adjacencyList[node.id] = node.dependencies + } + } + + // Check for cycles using DFS + var visited = Set() + var recursionStack = Set() + + func hasCycle(from nodeId: UUID) -> Bool { + visited.insert(nodeId) + recursionStack.insert(nodeId) + + if let dependencies = adjacencyList[nodeId] { + for dep in dependencies { + if !visited.contains(dep) { + if hasCycle(from: dep) { + return true + } + } else if recursionStack.contains(dep) { + return true + } + } + } + + recursionStack.remove(nodeId) + return false + } + + // Check each unvisited node + for nodeId in allNodes { + if !visited.contains(nodeId) { + if hasCycle(from: nodeId) { + throw BuildGraphError.cyclicDependency + } + } + } + } +} + +/// A build stage (FROM ... AS name). +/// +/// Design rationale: +/// - Represents a single FROM instruction and its operations +/// - Maintains operation order for correct execution +/// - Supports both named and anonymous stages +/// - Tracks dependencies on other stages +public struct BuildStage: Sendable, Equatable { + /// Unique identifier + public let id: UUID + + /// Stage name (FROM ... AS name) + public let name: String? + + /// Base image operation + public let base: ImageOperation + + /// Nodes in this stage (topologically sorted) + public let nodes: [BuildNode] + + /// Platform constraints for this stage + public let platform: Platform? + + public init( + id: UUID = UUID(), + name: String? = nil, + base: ImageOperation, + nodes: [BuildNode] = [], + platform: Platform? = nil + ) { + self.id = id + self.name = name + self.base = base + self.nodes = nodes + self.platform = platform + } + + /// All operations in this stage (including base) + public var operations: [any Operation] { + [base] + nodes.map { $0.operation } + } + + /// Find dependencies on other stages + public func stageDependencies() -> Set { + var deps = Set() + + for node in nodes { + // Check filesystem operations for stage references + if let fsOp = node.operation as? FilesystemOperation { + switch fsOp.source { + case .stage(let ref, _): + deps.insert(ref) + default: + break + } + } + + // Check mount sources + if let execOp = node.operation as? ExecOperation { + for mount in execOp.mounts { + if case .stage(let ref, _) = mount.source { + deps.insert(ref) + } + } + } + } + + return deps + } + + // MARK: - Validation + + func validate() throws { + // Stage-level validation is now done at the graph level + // to support cross-stage dependencies + } + +} + +/// A node in the build graph. +/// +/// Design rationale: +/// - Represents a single operation and its dependencies +/// - Immutable for safe concurrent access +/// - Tracks both data and execution dependencies +/// - Supports caching and incremental builds +public struct BuildNode: Sendable, Equatable { + /// Unique identifier + public let id: UUID + + /// The operation this node performs + public let operation: any Operation + + /// IDs of nodes this depends on + public let dependencies: Set + + /// Cache key for this operation + public let cacheKey: CacheKey? + + /// Execution constraints + public let constraints: Set + + public init( + id: UUID = UUID(), + operation: any Operation, + dependencies: Set = [], + cacheKey: CacheKey? = nil, + constraints: Set = [] + ) { + self.id = id + self.operation = operation + self.dependencies = dependencies + self.cacheKey = cacheKey + self.constraints = constraints + } + + // Custom Equatable implementation + public static func == (lhs: BuildNode, rhs: BuildNode) -> Bool { + // Compare by ID for node equality + lhs.id == rhs.id + } +} + +/// Build graph metadata. +public struct BuildGraphMetadata: Sendable { + /// Source file that generated this graph + public let sourceFile: String? + + /// Build context path + public let contextPath: String? + + /// Original frontend used (e.g., "dockerfile", "llb") + public let frontend: String? + + /// Frontend version + public let frontendVersion: String? + + /// Additional metadata + public let attributes: [String: AttributeValue] + + public init( + sourceFile: String? = nil, + contextPath: String? = nil, + frontend: String? = nil, + frontendVersion: String? = nil, + attributes: [String: AttributeValue] = [:] + ) { + self.sourceFile = sourceFile + self.contextPath = contextPath + self.frontend = frontend + self.frontendVersion = frontendVersion + self.attributes = attributes + } +} + +/// Cache key for operations. +/// +/// Design rationale: +/// - Content-addressed for reliable caching +/// - Includes all inputs that affect output +/// - Platform-aware for cross-compilation +public struct CacheKey: Hashable, Sendable { + /// Operation digest + public let operationDigest: Digest + + /// Input digests (from dependencies) + public let inputDigests: Set + + /// Platform (if platform-specific) + public let platform: Platform? + + /// Additional cache inputs + public let additionalInputs: [String: String] + + public init( + operationDigest: Digest, + inputDigests: Set = [], + platform: Platform? = nil, + additionalInputs: [String: String] = [:] + ) { + self.operationDigest = operationDigest + self.inputDigests = inputDigests + self.platform = platform + self.additionalInputs = additionalInputs + } + + /// Compute combined cache key + public var digest: Digest { + var data = Data() + data.append(operationDigest.bytes) + + for input in inputDigests.sorted(by: { $0.stringValue < $1.stringValue }) { + data.append(input.bytes) + } + + if let platform = platform { + data.append(contentsOf: platform.description.utf8) + } + + for (key, value) in additionalInputs.sorted(by: { $0.key < $1.key }) { + data.append(contentsOf: key.utf8) + data.append(contentsOf: value.utf8) + } + + do { + return try Digest.compute(data) + } catch { + // Fallback to a deterministic digest if computation fails + // This should never happen in practice + return try! Digest(algorithm: .sha256, bytes: Data(count: 32)) + } + } +} + +/// Execution constraints for nodes. +public enum Constraint: Hashable, Sendable { + /// Requires network access + case requiresNetwork + + /// Requires privileged execution + case requiresPrivileged + + /// Requires specific capability + case requiresCapability(String) + + /// Must run on specific platform + case requiresPlatform(Platform) + + /// Maximum execution time + case timeout(TimeInterval) + + /// Maximum memory + case memoryLimit(Int) + + /// CPU limit + case cpuLimit(Double) +} + +// MARK: - Errors + +public enum BuildGraphError: LocalizedError { + case duplicateStageName(String) + case cyclicDependency + case invalidDependency(UUID, UUID) + case stageNotFound(StageReference) + + public var errorDescription: String? { + switch self { + case .duplicateStageName(let name): + return "Duplicate stage name: '\(name)'" + case .cyclicDependency: + return "Build graph contains cyclic dependencies" + case .invalidDependency(let node, let dep): + return "Node \(node) has invalid dependency \(dep)" + case .stageNotFound(let ref): + return "Stage not found: \(ref)" + } + } +} + +// MARK: - Codable + +extension BuildGraph: Codable {} +extension BuildStage: Codable {} +extension BuildNode: Codable { + // Custom coding to handle type-erased Operation + enum CodingKeys: String, CodingKey { + case id + case operation + case dependencies + case cacheKey + case constraints + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(id, forKey: .id) + try container.encode(dependencies, forKey: .dependencies) + try container.encode(cacheKey, forKey: .cacheKey) + try container.encode(constraints, forKey: .constraints) + + // For operation, we need type information + // This is handled by SerializedOperation in IRCoder.swift + // For now, skip encoding the operation directly + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.id = try container.decode(UUID.self, forKey: .id) + self.dependencies = try container.decode(Set.self, forKey: .dependencies) + self.cacheKey = try container.decodeIfPresent(CacheKey.self, forKey: .cacheKey) + self.constraints = try container.decode(Set.self, forKey: .constraints) + + // For operation, we need a placeholder + // Real decoding is handled by SerializedNode in IRCoder.swift + self.operation = MetadataOperation(action: .setLabel(key: "placeholder", value: "placeholder")) + } +} +extension BuildGraphMetadata: Codable {} +extension CacheKey: Codable {} +extension Constraint: Codable {} diff --git a/Sources/NativeBuilder/ContainerBuildIR/Graph/GraphBuilder.swift b/Sources/NativeBuilder/ContainerBuildIR/Graph/GraphBuilder.swift new file mode 100644 index 00000000..a118c05c --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/Graph/GraphBuilder.swift @@ -0,0 +1,544 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildReporting +import ContainerizationOCI +import Foundation + +/// Errors that can occur during graph building +public enum GraphBuilderError: Error, LocalizedError { + case noActiveStage + case invalidOperation(String) + case missingDependency(UUID) + + public var errorDescription: String? { + switch self { + case .noActiveStage: + return "No active stage. Call stage() or scratch() first." + case .invalidOperation(let message): + return "Invalid operation: \(message)" + case .missingDependency(let id): + return "Missing dependency with ID: \(id)" + } + } +} + +/// Builder for constructing build graphs. +/// +/// Design rationale: +/// - Fluent API for easy graph construction +/// - Validates as you build to catch errors early +/// - Handles dependency resolution automatically +/// - Supports incremental construction +public final class GraphBuilder { + private var stages: [BuildStage] = [] + private var currentStage: StageBuilder? + private var buildArgs: [String: String] = [:] + private var targetPlatforms: Set = [] + private var metadata = BuildGraphMetadata() + private let graphAnalyzers: [any GraphAnalyzer] + private let stageAnalyzers: [any StageAnalyzer] + private let reporter: Reporter? + + /// Public initializer with optional reporter + public init(reporter: Reporter? = nil) { + self.graphAnalyzers = Self.defaultGraphAnalyzers + self.stageAnalyzers = Self.defaultStageAnalyzers + self.reporter = reporter + } + + /// Internal initializer for custom analyzers + internal init( + graphAnalyzers: [any GraphAnalyzer], + stageAnalyzers: [any StageAnalyzer], + reporter: Reporter? = nil + ) { + self.graphAnalyzers = graphAnalyzers + self.stageAnalyzers = stageAnalyzers + self.reporter = reporter + } + + /// Default graph analyzers + private nonisolated(unsafe) static let defaultGraphAnalyzers: [any GraphAnalyzer] = [ + DependencyAnalyzer(), + ValidatorAnalyzer(validator: StandardValidator()), + SemanticAnalyzer(), + ] + + /// Default stage analyzers (empty for now) + private nonisolated(unsafe) static let defaultStageAnalyzers: [any StageAnalyzer] = [] + + /// Start a new stage + @discardableResult + public func stage( + name: String? = nil, + from image: ImageReference, + platform: Platform? = nil + ) throws -> Self { + // Finish current stage if any + if let current = currentStage { + stages.append(try current.build()) + } + + // Report stage creation + if let reporter = reporter { + Task { + await reporter.report( + .irEvent( + context: ReportContext( + stageId: name, + description: "Creating stage \(name ?? "unnamed") from \(image)", + sourceMap: nil + ), + type: .stageAdded + )) + } + } + + // Create new stage + let imageOp = ImageOperation( + source: .registry(image), + platform: platform + ) + + currentStage = StageBuilder( + name: name, + base: imageOp, + platform: platform, + analyzers: stageAnalyzers, + reporter: reporter + ) + return self + } + + /// Start a stage from scratch + @discardableResult + public func scratch(name: String? = nil) throws -> Self { + if let current = currentStage { + stages.append(try current.build()) + } + + // Report stage creation + if let reporter = reporter { + Task { + await reporter.report( + .irEvent( + context: ReportContext( + stageId: name, + description: "Creating stage \(name ?? "unnamed") from scratch", + sourceMap: nil + ), + type: .stageAdded + )) + } + } + + let imageOp = ImageOperation(source: .scratch) + currentStage = StageBuilder( + name: name, + base: imageOp, + analyzers: stageAnalyzers, + reporter: reporter + ) + return self + } + + /// Add an operation to current stage + @discardableResult + public func add(_ operation: any Operation, dependsOn: [UUID] = []) throws -> Self { + guard let current = currentStage else { + throw GraphBuilderError.noActiveStage + } + + current.add(operation, dependsOn: Set(dependsOn)) + return self + } + + /// Add a RUN operation + @discardableResult + public func run( + _ command: String, + shell: Bool = true, + env: [String: String] = [:], + workdir: String? = nil, + user: User? = nil, + mounts: [Mount] = [], + network: NetworkMode = .default, + ) throws -> Self { + let cmd = shell ? Command.shell(command) : Command.exec(command.split(separator: " ").map(String.init)) + let envVars = env.map { (key: $0.key, value: EnvironmentValue.literal($0.value)) } + + let operation = ExecOperation( + command: cmd, + environment: Environment(envVars), + mounts: mounts, + workingDirectory: workdir, + user: user, + network: network, + ) + + return try add(operation) + } + + /// Add a COPY operation + @discardableResult + public func copy( + from source: FilesystemSource, + to destination: String, + chown: Ownership? = nil, + chmod: Permissions? = nil + ) throws -> Self { + let operation = FilesystemOperation( + action: .copy, + source: source, + destination: destination, + fileMetadata: FileMetadata( + ownership: chown, + permissions: chmod + ) + ) + + return try add(operation) + } + + /// Add COPY from context + @discardableResult + public func copyFromContext( + name: String = "default", + paths: [String], + to destination: String, + chown: Ownership? = nil, + chmod: Permissions? = nil + ) throws -> Self { + try copy( + from: .context(ContextSource(name: name, paths: paths)), + to: destination, + chown: chown, + chmod: chmod + ) + } + + /// Add COPY from stage + @discardableResult + public func copyFromStage( + _ stage: StageReference, + paths: [String], + to destination: String, + chown: Ownership? = nil, + chmod: Permissions? = nil + ) throws -> Self { + try copy( + from: .stage(stage, paths: paths), + to: destination, + chown: chown, + chmod: chmod + ) + } + + /// Set environment variable + @discardableResult + public func env(_ key: String, _ value: String) throws -> Self { + let operation = MetadataOperation( + action: .setEnv(key: key, value: .literal(value)) + ) + return try add(operation) + } + + /// Set working directory + @discardableResult + public func workdir(_ path: String) throws -> Self { + let operation = MetadataOperation(action: .setWorkdir(path: path)) + return try add(operation) + } + + /// Set user + @discardableResult + public func user(_ user: User) throws -> Self { + let operation = MetadataOperation(action: .setUser(user: user)) + return try add(operation) + } + + /// Add label + @discardableResult + public func label(_ key: String, _ value: String) throws -> Self { + let operation = MetadataOperation( + action: .setLabel(key: key, value: value) + ) + return try add(operation) + } + + /// Expose port + @discardableResult + public func expose(_ port: Int, protocolType: PortSpec.NetworkProtocol = .tcp) throws -> Self { + let operation = MetadataOperation( + action: .expose(port: PortSpec(port: port, protocol: protocolType)) + ) + return try add(operation) + } + + /// Set entrypoint + @discardableResult + public func entrypoint(_ command: Command) throws -> Self { + let operation = MetadataOperation(action: .setEntrypoint(command: command)) + return try add(operation) + } + + /// Set CMD + @discardableResult + public func cmd(_ command: Command) throws -> Self { + let operation = MetadataOperation(action: .setCmd(command: command)) + return try add(operation) + } + + /// Set healthcheck + @discardableResult + public func healthcheck( + test: HealthcheckTest, + interval: TimeInterval? = nil, + timeout: TimeInterval? = nil, + startPeriod: TimeInterval? = nil, + retries: Int? = nil + ) throws -> Self { + let healthcheck = Healthcheck( + test: test, + interval: interval, + timeout: timeout, + startPeriod: startPeriod, + retries: retries + ) + let operation = MetadataOperation(action: .setHealthcheck(healthcheck: healthcheck)) + return try add(operation) + } + + /// Add build argument + @discardableResult + public func arg(_ name: String, defaultValue: String? = nil) throws -> Self { + buildArgs[name] = defaultValue + let operation = MetadataOperation( + action: .declareArg(name: name, defaultValue: defaultValue) + ) + return try add(operation) + } + + /// Set target platforms + @discardableResult + public func platforms(_ platforms: Platform...) -> Self { + targetPlatforms = Set(platforms) + return self + } + + /// Set metadata + @discardableResult + public func metadata( + sourceFile: String? = nil, + contextPath: String? = nil, + frontend: String? = nil + ) -> Self { + if let sourceFile = sourceFile { + metadata = BuildGraphMetadata( + sourceFile: sourceFile, + contextPath: contextPath ?? metadata.contextPath, + frontend: frontend ?? metadata.frontend + ) + } + return self + } + + /// Build the final graph + public func build() throws -> BuildGraph { + // Report build start + if let reporter = reporter { + let stageCount = stages.count + Task { + await reporter.report( + .irEvent( + context: ReportContext( + description: "Building graph with \(stageCount) stages", + sourceMap: nil + ), + type: .graphStarted + )) + } + } + + // Add final stage if any + if let current = currentStage { + stages.append(try current.build()) + currentStage = nil + } + + // Create initial graph + var graph = try BuildGraph( + stages: stages, + buildArgs: buildArgs, + targetPlatforms: targetPlatforms, + metadata: metadata + ) + + // Create analysis context + let analysisContext = AnalysisContext(reporter: reporter) + + // Run all graph analyzers in sequence + for analyzer in graphAnalyzers { + let analyzerName = String(describing: type(of: analyzer)) + + // Report analyzer start + if let reporter = reporter { + Task { + await reporter.report( + .irEvent( + context: ReportContext( + description: "Running \(analyzerName)", + sourceMap: nil + ), + type: .analyzing + )) + } + } + + graph = try analyzer.analyze(graph, context: analysisContext) + } + + // Report completion + if let reporter = reporter { + let totalNodes = graph.stages.reduce(0) { $0 + $1.nodes.count } + let stageCount = graph.stages.count + Task { + await reporter.report( + .irEvent( + context: ReportContext( + description: "Graph built: \(stageCount) stages, \(totalNodes) nodes", + sourceMap: nil + ), + type: .graphCompleted + )) + } + } + + return graph + } +} + +/// Builder for individual stages. +private final class StageBuilder { + let name: String? + let base: ImageOperation + let platform: Platform? + private var nodes: [BuildNode] = [] + private var lastNodeId: UUID? + private let analyzers: [any StageAnalyzer] + private let reporter: Reporter? + + init(name: String?, base: ImageOperation, platform: Platform? = nil, analyzers: [any StageAnalyzer] = [], reporter: Reporter? = nil) { + self.name = name + self.base = base + self.platform = platform + self.analyzers = analyzers + self.reporter = reporter + } + + @discardableResult + func add(_ operation: any Operation, dependsOn: Set = []) -> UUID { + let node = BuildNode( + operation: operation, + dependencies: dependsOn + ) + + // Report node addition + if let reporter = reporter { + let stageId = name ?? "stage-\(node.id.uuidString.prefix(8))" + Task { + await reporter.report( + .irEvent( + context: ReportContext( + nodeId: node.id, + stageId: stageId, + description: "Added \(type(of: operation))", + sourceMap: nil + ), + type: .nodeAdded + )) + } + } + + nodes.append(node) + lastNodeId = node.id + return node.id + } + + func build() throws -> BuildStage { + var stage = BuildStage( + name: name, + base: base, + nodes: nodes, + platform: platform + ) + + // Create analysis context + let analysisContext = AnalysisContext(reporter: reporter) + + // Run stage analyzers + for analyzer in analyzers { + stage = try analyzer.analyze(stage, context: analysisContext) + } + + return stage + } +} + +// MARK: - Convenience Extensions + +extension GraphBuilder { + /// Create a simple single-stage build + public static func singleStage( + name: String? = nil, + from image: ImageReference, + platform: Platform? = nil, + reporter: Reporter? = nil, + _ configure: (GraphBuilder) throws -> Void + ) throws -> BuildGraph { + let builder = GraphBuilder(reporter: reporter) + if let platform = platform { + builder.platforms(platform) + } + try builder.stage(name: name, from: image, platform: platform) + try configure(builder) + return try builder.build() + } + + /// Create a multi-stage build + public static func multiStage( + reporter: Reporter? = nil, + _ configure: (GraphBuilder) throws -> Void + ) throws -> BuildGraph { + let builder = GraphBuilder(reporter: reporter) + try configure(builder) + return try builder.build() + } + + public func getStage(stageName: String) -> BuildStage? { + for s in self.stages { + if s.name == stageName { + return s + } + } + return nil + } + + public func getBuildArg(key: String) -> String? { + self.buildArgs[key] + } +} diff --git a/Sources/NativeBuilder/ContainerBuildIR/Graph/GraphTraversal.swift b/Sources/NativeBuilder/ContainerBuildIR/Graph/GraphTraversal.swift new file mode 100644 index 00000000..a8c48dd4 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/Graph/GraphTraversal.swift @@ -0,0 +1,288 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// Utilities for traversing and analyzing build graphs. +/// +/// Design rationale: +/// - Provides common graph algorithms (topological sort, dependency analysis) +/// - Supports both forward and reverse traversal +/// - Enables optimization passes and validation +public enum GraphTraversal { + + /// Perform topological sort on nodes in a stage. + /// + /// - Returns: Nodes in execution order + /// - Throws: If graph contains cycles + public static func topologicalSort(_ stage: BuildStage) throws -> [BuildNode] { + var sorted: [BuildNode] = [] + var visited = Set() + var visiting = Set() + + func visit(_ nodeId: UUID) throws { + if visiting.contains(nodeId) { + throw BuildGraphError.cyclicDependency + } + + if visited.contains(nodeId) { + return + } + + guard let node = stage.nodes.first(where: { $0.id == nodeId }) else { + return + } + + visiting.insert(nodeId) + + for dep in node.dependencies { + try visit(dep) + } + + visiting.remove(nodeId) + visited.insert(nodeId) + sorted.append(node) + } + + for node in stage.nodes { + try visit(node.id) + } + + return sorted + } + + /// Find all nodes that depend on a given node. + public static func findDependents( + of nodeId: UUID, + in stage: BuildStage + ) -> Set { + var dependents = Set() + + for node in stage.nodes { + if node.dependencies.contains(nodeId) { + dependents.insert(node.id) + // Recursively find transitive dependents + let transitive = findDependents(of: node.id, in: stage) + dependents.formUnion(transitive) + } + } + + return dependents + } + + /// Find all nodes that a given node depends on. + public static func findDependencies( + of nodeId: UUID, + in stage: BuildStage + ) -> Set { + guard let node = stage.nodes.first(where: { $0.id == nodeId }) else { + return [] + } + + var allDeps = node.dependencies + + for dep in node.dependencies { + let transitive = findDependencies(of: dep, in: stage) + allDeps.formUnion(transitive) + } + + return allDeps + } + + /// Find stages that a given stage depends on. + public static func findStageDependencies( + of stage: BuildStage, + in graph: BuildGraph + ) -> Set { + var stageDeps = Set() + + for dep in stage.stageDependencies() { + switch dep { + case .named(let name): + stageDeps.insert(name) + case .index(let idx): + if let depStage = graph.stage(at: idx), + let name = depStage.name + { + stageDeps.insert(name) + } + case .previous: + if let stageIndex = graph.stages.firstIndex(where: { $0.id == stage.id }), + stageIndex > 0, + let prevName = graph.stages[stageIndex - 1].name + { + stageDeps.insert(prevName) + } + } + } + + return stageDeps + } + + /// Perform a depth-first traversal of the graph. + public static func depthFirst( + stage: BuildStage, + visit: (BuildNode) throws -> Void + ) throws { + var visited = Set() + + func dfs(_ nodeId: UUID) throws { + guard !visited.contains(nodeId) else { return } + visited.insert(nodeId) + + guard let node = stage.nodes.first(where: { $0.id == nodeId }) else { + return + } + + // Visit dependencies first + for dep in node.dependencies { + try dfs(dep) + } + + try visit(node) + } + + // Visit all nodes, starting from roots but ensuring all nodes are visited + for node in stage.nodes { + try dfs(node.id) + } + } + + /// Find root nodes (nodes with no dependencies). + public static func findRoots(in stage: BuildStage) -> [BuildNode] { + stage.nodes.filter { $0.dependencies.isEmpty } + } + + /// Find leaf nodes (nodes with no dependents). + public static func findLeaves(in stage: BuildStage) -> [BuildNode] { + stage.nodes.filter { node in + !stage.nodes.contains { $0.dependencies.contains(node.id) } + } + } + + /// Calculate the critical path (longest path) through the graph. + public static func criticalPath(in stage: BuildStage) -> [BuildNode] { + // This is a simplified version - real implementation would + // consider execution times + var pathLengths = [UUID: Int]() + var nextInPath = [UUID: UUID?]() + + // Initialize all nodes + for node in stage.nodes { + pathLengths[node.id] = 1 + nextInPath[node.id] = nil + } + + // Calculate longest paths + if let sorted = try? topologicalSort(stage) { + for node in sorted.reversed() { + for dep in node.dependencies { + guard let nodeLength = pathLengths[node.id], + let depLength = pathLengths[dep] + else { + continue + } + + let newLength = nodeLength + 1 + if newLength > depLength { + pathLengths[dep] = newLength + nextInPath[dep] = node.id + } + } + } + } + + // Find the starting node with longest path + let start = pathLengths.max(by: { $0.value < $1.value })?.key + + // Build the path + var path: [BuildNode] = [] + var current = start + + while let nodeId = current, + let node = stage.nodes.first(where: { $0.id == nodeId }) + { + path.append(node) + current = nextInPath[nodeId] ?? nil + } + + return path + } +} + +/// Graph analysis results. +public struct GraphAnalysis { + /// Total number of operations + public let operationCount: Int + + /// Operations by type + public let operationsByType: [OperationKind: Int] + + /// Number of stages + public let stageCount: Int + + /// Stage dependencies + public let stageDependencies: [String: Set] + + /// Maximum depth of the graph + public let maxDepth: Int + + /// Critical path length + public let criticalPathLength: Int + + /// Parallelism opportunities (nodes that can run concurrently) + public let parallelismOpportunities: [[UUID]] +} + +extension BuildGraph { + /// Analyze the build graph structure. + public func analyze() -> GraphAnalysis { + var operationsByType = [OperationKind: Int]() + var stageDeps = [String: Set]() + var maxDepth = 0 + var criticalLength = 0 + + // Count operations and analyze stages + for stage in stages { + if let name = stage.name { + stageDeps[name] = GraphTraversal.findStageDependencies(of: stage, in: self) + } + + for op in stage.operations { + operationsByType[op.operationKind, default: 0] += 1 + } + + // Calculate depth + if let sorted = try? GraphTraversal.topologicalSort(stage) { + maxDepth = max(maxDepth, sorted.count) + } + + // Critical path + let critical = GraphTraversal.criticalPath(in: stage) + criticalLength = max(criticalLength, critical.count) + } + + return GraphAnalysis( + operationCount: stages.flatMap { $0.operations }.count, + operationsByType: operationsByType, + stageCount: stages.count, + stageDependencies: stageDeps, + maxDepth: maxDepth, + criticalPathLength: criticalLength, + parallelismOpportunities: [] // TODO: Implement + ) + } +} diff --git a/Sources/NativeBuilder/ContainerBuildIR/Operations/ExecOperation.swift b/Sources/NativeBuilder/ContainerBuildIR/Operations/ExecOperation.swift new file mode 100644 index 00000000..884f84f5 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/Operations/ExecOperation.swift @@ -0,0 +1,365 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// Represents a command execution operation (RUN in Dockerfile). +/// +/// Design rationale: +/// - Captures all execution context (env, mounts, user, workdir) +/// - Supports advanced features like secrets and SSH forwarding +/// - Shell vs exec form preserved for accurate execution +/// - Network and security controls built-in +public struct ExecOperation: Operation, Hashable { + public static let operationKind = OperationKind.exec + public var operationKind: OperationKind { Self.operationKind } + + /// The command to execute + public let command: Command + + /// Environment variables + public let environment: Environment + + /// Mounts (cache, bind, tmpfs, secret, ssh) + public let mounts: [Mount] + + /// Working directory + public let workingDirectory: String? + + /// User to run as + public let user: User? + + /// Network mode + public let network: NetworkMode + + /// Security options + public let security: SecurityOptions + + /// Operation metadata + public let metadata: OperationMetadata + + public init( + command: Command, + environment: Environment = .empty, + mounts: [Mount] = [], + workingDirectory: String? = nil, + user: User? = nil, + network: NetworkMode = .default, + security: SecurityOptions = .default, + metadata: OperationMetadata = OperationMetadata() + ) { + self.command = command + self.environment = environment + self.mounts = mounts + self.workingDirectory = workingDirectory + self.user = user + self.network = network + self.security = security + self.metadata = metadata + } + + public func accept(_ visitor: V) throws -> V.Result { + try visitor.visit(self) + } +} + +// MARK: - Command + +/// Represents a command to execute. +/// +/// Design rationale: +/// - Preserves shell vs exec form from Dockerfile +/// - Shell form uses default shell with command as string +/// - Exec form bypasses shell for direct execution +public enum Command: Hashable, Sendable { + /// Shell form: command is passed to shell + case shell(String) + + /// Exec form: direct execution without shell + case exec([String]) + + /// The command string for display + public var displayString: String { + switch self { + case .shell(let cmd): + return cmd + case .exec(let args): + return args.joined(separator: " ") + } + } + + /// Arguments for execution + public var arguments: [String] { + switch self { + case .shell(let cmd): + return ["/bin/sh", "-c", cmd] + case .exec(let args): + return args + } + } +} + +// MARK: - Environment + +/// Environment variables for execution. +/// +/// Design rationale: +/// - Preserves order for predictable overwrites +/// - Supports both literal values and build args +/// - Case-sensitive on Linux, case-insensitive on Windows +public struct Environment: Hashable, Sendable { + public let variables: [(key: String, value: EnvironmentValue)] + + public init(_ variables: [(key: String, value: EnvironmentValue)] = []) { + self.variables = variables + } + + public static let empty = Environment() + + /// Get effective environment as dictionary (last value wins) + public var effectiveEnvironment: [String: String] { + var result: [String: String] = [:] + for (key, value) in variables { + if case .literal(let str) = value { + result[key] = str + } + } + return result + } +} + +// Custom Hashable conformance for Environment +extension Environment { + public static func == (lhs: Environment, rhs: Environment) -> Bool { + guard lhs.variables.count == rhs.variables.count else { return false } + for (index, (lkey, lvalue)) in lhs.variables.enumerated() { + let (rkey, rvalue) = rhs.variables[index] + if lkey != rkey || lvalue != rvalue { + return false + } + } + return true + } + + public func hash(into hasher: inout Hasher) { + for (key, value) in variables { + hasher.combine(key) + hasher.combine(value) + } + } +} + +/// Environment variable value. +public enum EnvironmentValue: Hashable, Sendable { + /// Literal string value + case literal(String) + + /// Reference to build argument + case buildArg(String) + + /// Expansion with default + case expansion(name: String, default: String?) +} + +// MARK: - Mounts + +/// Represents a mount in the container. +/// +/// Design rationale: +/// - Type-safe mount specifications +/// - Supports all Dockerfile mount types +/// - Extensible for future mount types +public struct Mount: Hashable, Sendable { + public let type: MountType + public let target: String? + public let envTarget: String? + public let source: MountSource? + public let options: MountOptions + + public init( + type: MountType, + target: String? = nil, + envTarget: String? = nil, + source: MountSource? = nil, + options: MountOptions = MountOptions() + ) { + self.type = type + self.target = target + self.envTarget = envTarget + self.source = source + self.options = options + } +} + +/// Type of mount. +public enum MountType: String, Hashable, Sendable { + case bind + case cache + case tmpfs + case secret + case ssh +} + +/// Source of mount data. +public enum MountSource: Hashable, Sendable { + /// Local path + case local(String) + + /// From another stage + case stage(StageReference, path: String) + + /// From image + case image(ImageReference, path: String) + + /// Build context + case context(String, path: String) + + /// Secret by ID + case secret(String) + + /// SSH agent socket + case sshAgent +} + +/// Mount options. +public struct MountOptions: Hashable, Sendable { + public let readOnly: Bool + public let uid: UInt32? + public let gid: UInt32? + public let mode: UInt32? + public let size: UInt32? // For tmpfs + public let sharing: SharingMode? // For cache mounts + public let required: Bool? + + public init( + readOnly: Bool = false, + uid: UInt32? = nil, + gid: UInt32? = nil, + mode: UInt32? = nil, + size: UInt32? = nil, + sharing: SharingMode? = nil, + required: Bool? = nil, + ) { + self.readOnly = readOnly + self.uid = uid + self.gid = gid + self.mode = mode + self.size = size + self.sharing = sharing + self.required = required + } +} + +// MARK: - User + +/// User specification for command execution. +public enum User: Hashable, Sendable { + /// User by name + case named(String) + + /// User by UID + case uid(UInt32) + + /// User and group + case userGroup(user: String, group: String) + + /// UID and GID + case uidGid(uid: UInt32, gid: UInt32) +} + +// MARK: - Network + +/// Network mode for execution. +public enum NetworkMode: String, Hashable, Sendable { + /// Default network + case `default` + + /// No network + case none + + /// Host network + case host +} + +// MARK: - Security + +/// Security options for execution. +public struct SecurityOptions: Hashable, Sendable { + public let privileged: Bool + public let capabilities: SecurityCapabilities? + public let seccompProfile: String? + public let apparmorProfile: String? + public let noNewPrivileges: Bool + + public init( + privileged: Bool = false, + capabilities: SecurityCapabilities? = nil, + seccompProfile: String? = nil, + apparmorProfile: String? = nil, + noNewPrivileges: Bool = true + ) { + self.privileged = privileged + self.capabilities = capabilities + self.seccompProfile = seccompProfile + self.apparmorProfile = apparmorProfile + self.noNewPrivileges = noNewPrivileges + } + + public static let `default` = SecurityOptions() +} + +/// Linux capabilities. +public struct SecurityCapabilities: Hashable, Sendable { + public let add: Set + public let drop: Set + + public init(add: Set = [], drop: Set = []) { + self.add = add + self.drop = drop + } +} + +// MARK: - Codable + +extension ExecOperation: Codable {} +extension Command: Codable {} +extension Environment: Codable { + private struct EnvVar: Codable { + let key: String + let value: EnvironmentValue + } + + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + let envVars = try container.decode([EnvVar].self) + self.variables = envVars.map { ($0.key, $0.value) } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + let envVars = variables.map { EnvVar(key: $0.key, value: $0.value) } + try container.encode(envVars) + } +} +extension EnvironmentValue: Codable {} +extension Mount: Codable {} +extension MountType: Codable {} +extension MountSource: Codable {} +extension MountOptions: Codable {} +extension User: Codable {} +extension NetworkMode: Codable {} +extension SecurityOptions: Codable {} +extension SecurityCapabilities: Codable {} diff --git a/Sources/NativeBuilder/ContainerBuildIR/Operations/FilesystemOperation.swift b/Sources/NativeBuilder/ContainerBuildIR/Operations/FilesystemOperation.swift new file mode 100644 index 00000000..231f44bd --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/Operations/FilesystemOperation.swift @@ -0,0 +1,313 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// Represents filesystem operations (COPY, ADD, etc.). +/// +/// Design rationale: +/// - Unified handling of all filesystem modifications +/// - Preserves source context (local, stage, URL) +/// - Supports advanced features like ownership and permissions +/// - Extensible for future filesystem operations +public struct FilesystemOperation: Operation, Hashable { + public static let operationKind = OperationKind.filesystem + public var operationKind: OperationKind { Self.operationKind } + + /// The filesystem action to perform + public let action: FilesystemAction + + /// Source of the files + public let source: FilesystemSource + + /// Destination path + public let destination: String + + /// File metadata (ownership, permissions) + public let fileMetadata: FileMetadata + + /// Copy options + public let options: FilesystemOptions + + /// Operation metadata + public let metadata: OperationMetadata + + public init( + action: FilesystemAction, + source: FilesystemSource, + destination: String, + fileMetadata: FileMetadata = FileMetadata(), + options: FilesystemOptions = FilesystemOptions(), + metadata: OperationMetadata = OperationMetadata() + ) { + self.action = action + self.source = source + self.destination = destination + self.fileMetadata = fileMetadata + self.options = options + self.metadata = metadata + } + + public func accept(_ visitor: V) throws -> V.Result { + try visitor.visit(self) + } +} + +// MARK: - Filesystem Action + +/// Type of filesystem action. +/// +/// Design rationale: +/// - Covers all Dockerfile filesystem operations +/// - Extensible for future operations +/// - Clear semantics for each action +public enum FilesystemAction: String, Hashable, Sendable { + /// Copy files (COPY instruction) + case copy + + /// Add files with URL/tar support (ADD instruction) + case add + + /// Remove files + case remove + + /// Create directory + case mkdir + + /// Create symlink + case symlink + + /// Create hard link + case hardlink +} + +// MARK: - Filesystem Source + +/// Source of filesystem content. +/// +/// Design rationale: +/// - Type-safe source specifications +/// - Supports all Dockerfile source types +/// - Extensible for future sources +public enum FilesystemSource: Hashable, Sendable { + /// Files from build context + case context(ContextSource) + + /// Files from another stage + case stage(StageReference, paths: [String]) + + /// Files from an image + case image(ImageReference, paths: [String]) + + /// URL to download + case url(URL) + + /// Git repository + case git(GitSource) + + /// Inline content + case inline(Data) + + /// Empty/scratch + case scratch +} + +/// Build context source. +public struct ContextSource: Hashable, Sendable { + /// Name of the context + public let name: String + + /// Paths relative to context root + public let paths: [String] + + /// Include patterns (if empty, all files match) + public let includes: [String] + + /// Exclude patterns + public let excludes: [String] + + public init(name: String = "default", paths: [String], includes: [String] = [], excludes: [String] = []) { + self.name = name + self.paths = paths + self.includes = includes + self.excludes = excludes + } +} + +/// Git repository source. +public struct GitSource: Hashable, Sendable { + public let repository: String + public let reference: String? // branch, tag, commit + public let submodules: Bool + + public init(repository: String, reference: String? = nil, submodules: Bool = false) { + self.repository = repository + self.reference = reference + self.submodules = submodules + } +} + +// MARK: - File Metadata + +/// Metadata for created/modified files. +/// +/// Design rationale: +/// - Captures all file attributes +/// - Platform-aware (Unix permissions vs Windows ACLs) +/// - Preserves source metadata by default +public struct FileMetadata: Hashable, Sendable { + /// File ownership + public let ownership: Ownership? + + /// File permissions (Unix mode) + public let permissions: Permissions? + + /// Timestamps + public let timestamps: Timestamps? + + /// Extended attributes + public let xattrs: [String: Data] + + public init( + ownership: Ownership? = nil, + permissions: Permissions? = nil, + timestamps: Timestamps? = nil, + xattrs: [String: Data] = [:] + ) { + self.ownership = ownership + self.permissions = permissions + self.timestamps = timestamps + self.xattrs = xattrs + } +} + +public enum OwnershipID: Hashable, Sendable { + /// Numeric UID/GID + case numeric(id: UInt32) + + /// Named user/group + case named(id: String) +} + +/// File ownership. +public struct Ownership: Hashable, Sendable { + public let userID: OwnershipID? + public let groupID: OwnershipID? + + public init(user: OwnershipID? = nil, group: OwnershipID? = nil) { + self.userID = user + self.groupID = group + } +} + +/// File permissions. +public enum Permissions: Hashable, Sendable { + /// Unix mode (e.g., 0755) + case mode(UInt32) + + /// Symbolic (e.g., "u+x") + case symbolic(String) + + /// Preserve from source + case preserve +} + +/// File timestamps. +public struct Timestamps: Hashable, Sendable { + public let created: Date? + public let modified: Date? + public let accessed: Date? + + public init(created: Date? = nil, modified: Date? = nil, accessed: Date? = nil) { + self.created = created + self.modified = modified + self.accessed = accessed + } +} + +// MARK: - Filesystem Options + +/// Options for filesystem operations. +/// +/// Design rationale: +/// - Controls operation behavior +/// - Platform-specific handling +/// - Performance optimizations +public struct FilesystemOptions: Hashable, Sendable { + /// Follow symlinks in source + public let followSymlinks: Bool + + /// Preserve timestamps + public let preserveTimestamps: Bool + + /// Merge directories (don't replace) + public let merge: Bool + + /// Create parent directories + public let createParents: Bool + + /// For ADD: auto-extract archives + public let extractArchives: Bool + + /// Copy strategy + public let copyStrategy: CopyStrategy + + public init( + followSymlinks: Bool = true, + preserveTimestamps: Bool = false, + merge: Bool = true, + createParents: Bool = true, + extractArchives: Bool = true, + copyStrategy: CopyStrategy = .auto + ) { + self.followSymlinks = followSymlinks + self.preserveTimestamps = preserveTimestamps + self.merge = merge + self.createParents = createParents + self.extractArchives = extractArchives + self.copyStrategy = copyStrategy + } +} + +/// Strategy for copying files. +public enum CopyStrategy: String, Hashable, Sendable { + /// Automatically choose best method + case auto + + /// Copy-on-write if available + case cow + + /// Hard link if possible + case hardlink + + /// Always full copy + case copy +} + +// MARK: - Codable + +extension FilesystemOperation: Codable {} +extension FilesystemAction: Codable {} +extension FilesystemSource: Codable {} +extension ContextSource: Codable {} +extension GitSource: Codable {} +extension FileMetadata: Codable {} +extension OwnershipID: Codable {} +extension Ownership: Codable {} +extension Permissions: Codable {} +extension Timestamps: Codable {} +extension FilesystemOptions: Codable {} +extension CopyStrategy: Codable {} diff --git a/Sources/NativeBuilder/ContainerBuildIR/Operations/ImageOperation.swift b/Sources/NativeBuilder/ContainerBuildIR/Operations/ImageOperation.swift new file mode 100644 index 00000000..41aacd2f --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/Operations/ImageOperation.swift @@ -0,0 +1,173 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOCI +import Foundation + +/// Represents base image operations (FROM in Dockerfile). +/// +/// Design rationale: +/// - Handles both image pulls and scratch images +/// - Platform-aware for multi-arch support +/// - Supports image verification and policy +public struct ImageOperation: Operation, Hashable, Equatable { + public static let operationKind = OperationKind.image + public var operationKind: OperationKind { Self.operationKind } + + /// Image source + public let source: ImageSource + + /// Target platform + public let platform: Platform? + + /// Pull policy + public let pullPolicy: PullPolicy + + /// Image verification + public let verification: ImageVerification? + + /// Operation metadata + public let metadata: OperationMetadata + + public init( + source: ImageSource, + platform: Platform? = nil, + pullPolicy: PullPolicy = .ifNotPresent, + verification: ImageVerification? = nil, + metadata: OperationMetadata = OperationMetadata() + ) { + self.source = source + self.platform = platform + self.pullPolicy = pullPolicy + self.verification = verification + self.metadata = metadata + } + + public func accept(_ visitor: V) throws -> V.Result { + try visitor.visit(self) + } +} + +// MARK: - Image Source + +/// Source of base image. +/// +/// Design rationale: +/// - Supports registry images and scratch +/// - Extensible for future image sources +/// - Clear semantics for each source type +public enum ImageSource: Hashable, Sendable { + /// Image from registry + case registry(ImageReference) + + /// Empty image (FROM scratch) + case scratch + + /// Local OCI layout + case ociLayout(path: String, tag: String?) + + /// Tarball + case tarball(path: String) +} + +// MARK: - Pull Policy + +/// Policy for pulling images. +/// +/// Design rationale: +/// - Matches Kubernetes/Docker semantics +/// - Allows optimization vs freshness tradeoffs +public enum PullPolicy: String, Hashable, Sendable { + /// Always pull + case always + + /// Pull if not present locally + case ifNotPresent + + /// Never pull (must exist locally) + case never +} + +// MARK: - Image Verification + +/// Image verification requirements. +/// +/// Design rationale: +/// - Supports multiple verification methods +/// - Extensible for new verification types +/// - Policy-based for enterprise requirements +public struct ImageVerification: Hashable, Sendable { + /// Verification method + public let method: VerificationMethod + + /// Required signatures + public let requiredSignatures: Int + + /// Trusted keys/identities + public let trustedKeys: Set + + public init( + method: VerificationMethod, + requiredSignatures: Int = 1, + trustedKeys: Set = [] + ) { + self.method = method + self.requiredSignatures = requiredSignatures + self.trustedKeys = trustedKeys + } +} + +/// Verification method. +public enum VerificationMethod: String, Hashable, Sendable { + /// No verification + case none + + /// Verify digest only + case digest + + /// Cosign signatures + case cosign + + /// Notary v2 + case notary + + /// In-toto attestations + case intoto +} + +/// Trusted key for verification. +public enum TrustedKey: Hashable, Sendable { + /// Public key + case publicKey(Data) + + /// Key ID (for key servers) + case keyID(String) + + /// Certificate + case certificate(Data) + + /// OIDC identity + case oidcIdentity(issuer: String, subject: String) +} + +// MARK: - Codable + +extension ImageOperation: Codable {} +extension ImageSource: Codable {} +extension PullPolicy: Codable {} +extension ImageVerification: Codable {} +extension VerificationMethod: Codable {} +extension TrustedKey: Codable {} diff --git a/Sources/NativeBuilder/ContainerBuildIR/Operations/MetadataOperation.swift b/Sources/NativeBuilder/ContainerBuildIR/Operations/MetadataOperation.swift new file mode 100644 index 00000000..dde800fa --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/Operations/MetadataOperation.swift @@ -0,0 +1,454 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// Represents metadata operations (ENV, LABEL, ARG, etc.). +/// +/// Design rationale: +/// - Unified handling of all metadata modifications +/// - No filesystem changes, only configuration +/// - Supports both build-time and runtime metadata +public struct MetadataOperation: Operation, Hashable { + public static let operationKind = OperationKind.metadata + public var operationKind: OperationKind { Self.operationKind } + + /// Type of metadata operation + public let action: MetadataAction + + /// Operation metadata + public let metadata: OperationMetadata + + public init( + action: MetadataAction, + metadata: OperationMetadata = OperationMetadata() + ) { + self.action = action + self.metadata = metadata + } + + public func accept(_ visitor: V) throws -> V.Result { + try visitor.visit(self) + } +} + +// MARK: - Metadata Action + +/// Type of metadata modification. +/// +/// Design rationale: +/// - Each action is self-contained with its data +/// - Clear separation between build and runtime metadata +/// - Extensible for future metadata types +public enum MetadataAction: Sendable { + /// Set environment variable (ENV) + case setEnv(key: String, value: EnvironmentValue) + + /// Set multiple environment variables + case setEnvBatch([(key: String, value: EnvironmentValue)]) + + /// Set label (LABEL) + case setLabel(key: String, value: String) + + /// Set multiple labels + case setLabelBatch([String: String]) + + /// Define build argument (ARG) + case declareArg(name: String, defaultValue: String?) + + /// Set exposed port (EXPOSE) + case expose(port: PortSpec) + + /// Set working directory (WORKDIR) + case setWorkdir(path: String) + + /// Set user (USER) + case setUser(user: User) + + /// Set entrypoint (ENTRYPOINT) + case setEntrypoint(command: Command) + + /// Set default command (CMD) + case setCmd(command: Command) + + /// Set shell (SHELL) + case setShell(shell: [String]) + + /// Set healthcheck (HEALTHCHECK) + case setHealthcheck(healthcheck: Healthcheck?) + + /// Set stop signal (STOPSIGNAL) + case setStopSignal(signal: String) + + /// Add volume (VOLUME) + case addVolume(path: String) + + /// Add onbuild trigger (ONBUILD) + case addOnBuild(instruction: String) +} + +// MARK: - Port Specification + +/// Port exposure specification. +/// +/// Design rationale: +/// - Supports TCP/UDP/SCTP +/// - Range support for multiple ports +/// - Documentation via description +public struct PortSpec: Hashable, Sendable { + public enum NetworkProtocol: String, Hashable, Sendable { + case tcp + case udp + case sctp + } + + /// Port number or range start + public let port: Int + + /// Range end (if range) + public let endPort: Int? + + /// Protocol + public let `protocol`: NetworkProtocol + + /// Human-readable description + public let description: String? + + public init( + port: Int, + endPort: Int? = nil, + protocol: NetworkProtocol = .tcp, + description: String? = nil + ) { + self.port = port + self.endPort = endPort + self.`protocol` = `protocol` + self.description = description + } + + /// String representation (e.g., "80/tcp", "8000-8100/udp") + public var stringValue: String { + guard let endPort = endPort else { + return "\(port)/\(`protocol`.rawValue)" + } + return "\(port)-\(endPort)/\(`protocol`.rawValue)" + } +} + +// MARK: - Healthcheck + +/// Container healthcheck configuration. +/// +/// Design rationale: +/// - Matches Docker/OCI healthcheck spec +/// - Flexible timing configuration +/// - Supports disabling inherited healthchecks +public struct Healthcheck: Hashable, Sendable { + /// Test command + public let test: HealthcheckTest + + /// Time between checks + public let interval: TimeInterval? + + /// Timeout for each check + public let timeout: TimeInterval? + + /// Initial delay before first check + public let startPeriod: TimeInterval? + + /// Number of retries before unhealthy + public let retries: Int? + + public init( + test: HealthcheckTest, + interval: TimeInterval? = nil, + timeout: TimeInterval? = nil, + startPeriod: TimeInterval? = nil, + retries: Int? = nil + ) { + self.test = test + self.interval = interval + self.timeout = timeout + self.startPeriod = startPeriod + self.retries = retries + } +} + +/// Healthcheck test specification. +public enum HealthcheckTest: Hashable, Sendable { + /// No healthcheck (NONE) + case none + + /// Command to run (CMD) + case command(Command) + + /// Command with shell (CMD-SHELL) + case shell(String) +} + +// MARK: - Hashable & Equatable + +extension MetadataAction: Hashable, Equatable { + public static func == (lhs: MetadataAction, rhs: MetadataAction) -> Bool { + switch (lhs, rhs) { + case (.setEnv(let lk, let lv), .setEnv(let rk, let rv)): + return lk == rk && lv == rv + case (.setEnvBatch(let l), .setEnvBatch(let r)): + guard l.count == r.count else { return false } + for (index, (lk, lv)) in l.enumerated() { + let (rk, rv) = r[index] + if lk != rk || lv != rv { return false } + } + return true + case (.setLabel(let lk, let lv), .setLabel(let rk, let rv)): + return lk == rk && lv == rv + case (.setLabelBatch(let l), .setLabelBatch(let r)): + return l == r + case (.declareArg(let ln, let ld), .declareArg(let rn, let rd)): + return ln == rn && ld == rd + case (.expose(let l), .expose(let r)): + return l == r + case (.setWorkdir(let l), .setWorkdir(let r)): + return l == r + case (.setUser(let l), .setUser(let r)): + return l == r + case (.setEntrypoint(let l), .setEntrypoint(let r)): + return l == r + case (.setCmd(let l), .setCmd(let r)): + return l == r + case (.setShell(let l), .setShell(let r)): + return l == r + case (.setHealthcheck(let l), .setHealthcheck(let r)): + return l == r + case (.setStopSignal(let l), .setStopSignal(let r)): + return l == r + case (.addVolume(let l), .addVolume(let r)): + return l == r + case (.addOnBuild(let l), .addOnBuild(let r)): + return l == r + default: + return false + } + } + + public func hash(into hasher: inout Hasher) { + switch self { + case .setEnv(let key, let value): + hasher.combine(0) + hasher.combine(key) + hasher.combine(value) + case .setEnvBatch(let vars): + hasher.combine(1) + for (key, value) in vars { + hasher.combine(key) + hasher.combine(value) + } + case .setLabel(let key, let value): + hasher.combine(2) + hasher.combine(key) + hasher.combine(value) + case .setLabelBatch(let labels): + hasher.combine(3) + hasher.combine(labels) + case .declareArg(let name, let defaultValue): + hasher.combine(4) + hasher.combine(name) + hasher.combine(defaultValue) + case .expose(let port): + hasher.combine(5) + hasher.combine(port) + case .setWorkdir(let path): + hasher.combine(6) + hasher.combine(path) + case .setUser(let user): + hasher.combine(7) + hasher.combine(user) + case .setEntrypoint(let command): + hasher.combine(8) + hasher.combine(command) + case .setCmd(let command): + hasher.combine(9) + hasher.combine(command) + case .setShell(let shell): + hasher.combine(10) + hasher.combine(shell) + case .setHealthcheck(let healthcheck): + hasher.combine(11) + hasher.combine(healthcheck) + case .setStopSignal(let signal): + hasher.combine(12) + hasher.combine(signal) + case .addVolume(let path): + hasher.combine(13) + hasher.combine(path) + case .addOnBuild(let instruction): + hasher.combine(14) + hasher.combine(instruction) + } + } +} + +// MARK: - Codable + +extension MetadataOperation: Codable {} +extension MetadataAction: Codable { + private enum CodingKeys: String, CodingKey { + case type + case key + case value + case envVars + case labels + case name + case defaultValue + case port + case path + case user + case command + case shell + case healthcheck + case signal + case instruction + } + + private struct EnvVar: Codable { + let key: String + let value: EnvironmentValue + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .setEnv(let key, let value): + try container.encode("setEnv", forKey: .type) + try container.encode(key, forKey: .key) + try container.encode(value, forKey: .value) + case .setEnvBatch(let vars): + try container.encode("setEnvBatch", forKey: .type) + let envVars = vars.map { EnvVar(key: $0.key, value: $0.value) } + try container.encode(envVars, forKey: .envVars) + case .setLabel(let key, let value): + try container.encode("setLabel", forKey: .type) + try container.encode(key, forKey: .key) + try container.encode(value, forKey: .value) + case .setLabelBatch(let labels): + try container.encode("setLabelBatch", forKey: .type) + try container.encode(labels, forKey: .labels) + case .declareArg(let name, let defaultValue): + try container.encode("declareArg", forKey: .type) + try container.encode(name, forKey: .name) + try container.encode(defaultValue, forKey: .defaultValue) + case .expose(let port): + try container.encode("expose", forKey: .type) + try container.encode(port, forKey: .port) + case .setWorkdir(let path): + try container.encode("setWorkdir", forKey: .type) + try container.encode(path, forKey: .path) + case .setUser(let user): + try container.encode("setUser", forKey: .type) + try container.encode(user, forKey: .user) + case .setEntrypoint(let command): + try container.encode("setEntrypoint", forKey: .type) + try container.encode(command, forKey: .command) + case .setCmd(let command): + try container.encode("setCmd", forKey: .type) + try container.encode(command, forKey: .command) + case .setShell(let shell): + try container.encode("setShell", forKey: .type) + try container.encode(shell, forKey: .shell) + case .setHealthcheck(let healthcheck): + try container.encode("setHealthcheck", forKey: .type) + try container.encode(healthcheck, forKey: .healthcheck) + case .setStopSignal(let signal): + try container.encode("setStopSignal", forKey: .type) + try container.encode(signal, forKey: .signal) + case .addVolume(let path): + try container.encode("addVolume", forKey: .type) + try container.encode(path, forKey: .path) + case .addOnBuild(let instruction): + try container.encode("addOnBuild", forKey: .type) + try container.encode(instruction, forKey: .instruction) + } + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let type = try container.decode(String.self, forKey: .type) + + switch type { + case "setEnv": + let key = try container.decode(String.self, forKey: .key) + let value = try container.decode(EnvironmentValue.self, forKey: .value) + self = .setEnv(key: key, value: value) + case "setEnvBatch": + let envVars = try container.decode([EnvVar].self, forKey: .envVars) + self = .setEnvBatch(envVars.map { ($0.key, $0.value) }) + case "setLabel": + let key = try container.decode(String.self, forKey: .key) + let value = try container.decode(String.self, forKey: .value) + self = .setLabel(key: key, value: value) + case "setLabelBatch": + let labels = try container.decode([String: String].self, forKey: .labels) + self = .setLabelBatch(labels) + case "declareArg": + let name = try container.decode(String.self, forKey: .name) + let defaultValue = try container.decodeIfPresent(String.self, forKey: .defaultValue) + self = .declareArg(name: name, defaultValue: defaultValue) + case "expose": + let port = try container.decode(PortSpec.self, forKey: .port) + self = .expose(port: port) + case "setWorkdir": + let path = try container.decode(String.self, forKey: .path) + self = .setWorkdir(path: path) + case "setUser": + let user = try container.decode(User.self, forKey: .user) + self = .setUser(user: user) + case "setEntrypoint": + let command = try container.decode(Command.self, forKey: .command) + self = .setEntrypoint(command: command) + case "setCmd": + let command = try container.decode(Command.self, forKey: .command) + self = .setCmd(command: command) + case "setShell": + let shell = try container.decode([String].self, forKey: .shell) + self = .setShell(shell: shell) + case "setHealthcheck": + let healthcheck = try container.decodeIfPresent(Healthcheck.self, forKey: .healthcheck) + self = .setHealthcheck(healthcheck: healthcheck) + case "setStopSignal": + let signal = try container.decode(String.self, forKey: .signal) + self = .setStopSignal(signal: signal) + case "addVolume": + let path = try container.decode(String.self, forKey: .path) + self = .addVolume(path: path) + case "addOnBuild": + let instruction = try container.decode(String.self, forKey: .instruction) + self = .addOnBuild(instruction: instruction) + default: + throw DecodingError.dataCorruptedError(forKey: .type, in: container, debugDescription: "Unknown MetadataAction type: \(type)") + } + } +} +extension PortSpec: Codable { + enum CodingKeys: String, CodingKey { + case port + case endPort + case `protocol` + case description + } +} +extension PortSpec.NetworkProtocol: Codable {} +extension Healthcheck: Codable {} +extension HealthcheckTest: Codable {} diff --git a/Sources/NativeBuilder/ContainerBuildIR/Operations/Operation.swift b/Sources/NativeBuilder/ContainerBuildIR/Operations/Operation.swift new file mode 100644 index 00000000..fadb30a9 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/Operations/Operation.swift @@ -0,0 +1,288 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOCI +import Crypto +import Foundation + +/// Errors that can occur during operation processing. +public enum OperationError: LocalizedError { + case encodingFailed(String) + case digestComputationFailed(Error) + + public var errorDescription: String? { + switch self { + case .encodingFailed(let details): + return "Failed to encode operation data: \(details)" + case .digestComputationFailed(let error): + return "Failed to compute operation digest: \(error.localizedDescription)" + } + } +} + +/// Core operation protocol that all build operations must conform to. +/// +/// Design rationale: +/// - Protocol-based design allows extending with new operations without modifying existing code +/// - Each operation is self-contained with all necessary data +/// - Operations are immutable for thread safety and predictable behavior +/// - Visitor pattern support for traversal and transformation +public protocol Operation: Sendable { + /// Common metadata associated with this operation. + var metadata: OperationMetadata { get } + + /// Unique identifier for this operation type + static var operationKind: OperationKind { get } + + /// Instance identifier + var operationKind: OperationKind { get } + + /// Accept a visitor for traversal/transformation + func accept(_ visitor: V) throws -> V.Result +} + +/// Identifies the type of operation. +/// +/// Design rationale: +/// - String-based for extensibility (third-party operations) +/// - Comparable for consistent ordering +/// - Provides namespace for operation types +public struct OperationKind: RawRepresentable, Hashable, Sendable { + public let rawValue: String + + public init(rawValue: String) { + self.rawValue = rawValue + } + + // Core operation kinds + public static let exec = OperationKind(rawValue: "core.exec") + public static let filesystem = OperationKind(rawValue: "core.filesystem") + public static let image = OperationKind(rawValue: "core.image") + public static let metadata = OperationKind(rawValue: "core.metadata") + public static let mount = OperationKind(rawValue: "core.mount") +} + +/// Visitor pattern for operation traversal. +/// +/// Design rationale: +/// - Type-safe traversal without casting +/// - Extensible for new operations +/// - Supports both read-only traversal and transformation +public protocol OperationVisitor { + associatedtype Result + + func visit(_ operation: ExecOperation) throws -> Result + func visit(_ operation: FilesystemOperation) throws -> Result + func visit(_ operation: ImageOperation) throws -> Result + func visit(_ operation: MetadataOperation) throws -> Result + + /// Default handler for unknown operations + func visitUnknown(_ operation: any Operation) throws -> Result +} + +/// Base class for operations providing common functionality. +/// +/// Design rationale: +/// - While we prefer protocols, a base class here provides default implementations +/// - Reduces boilerplate for operation implementations +/// - Still allows protocol-based extension +@available(*, unavailable, message: "Use specific operation types instead") +open class BaseOperation: @unchecked Sendable, Operation { + public var metadata: OperationMetadata + + public init(metadata: OperationMetadata) { + self.metadata = metadata + } + + public static var operationKind: OperationKind { + // This class is unavailable for use, but we need to provide a value + // for protocol conformance. This should never be called in practice. + OperationKind(rawValue: "base.unavailable") + } + public var operationKind: OperationKind { Self.operationKind } + + public func accept(_ visitor: V) throws -> V.Result { + try visitor.visitUnknown(self) + } +} + +// MARK: - Operation Metadata + +/// Common metadata that can be attached to any operation. +/// +/// Design rationale: +/// - Extensible key-value storage for future needs +/// - Strongly-typed for common attributes +/// - Preserves unknown attributes for forward compatibility +public struct OperationMetadata: Sendable, Hashable { + /// Human-readable comment/description + public let comment: String? + + /// Source location (file:line) where this operation was defined + public let sourceLocation: SourceLocation? + + /// Platform constraints for this operation + public let platforms: Set? + + /// Cache configuration + public let cacheConfig: CacheConfig? + + /// The retry policy for this specific operation. + public let retryPolicy: RetryPolicy + + /// Additional attributes + public let attributes: [String: AttributeValue] + + public init( + comment: String? = nil, + sourceLocation: SourceLocation? = nil, + platforms: Set? = nil, + cacheConfig: CacheConfig? = nil, + retryPolicy: RetryPolicy = RetryPolicy(maxRetries: 0), + attributes: [String: AttributeValue] = [:] + ) { + self.comment = comment + self.sourceLocation = sourceLocation + self.platforms = platforms + self.cacheConfig = cacheConfig + self.retryPolicy = retryPolicy + self.attributes = attributes + } +} + +/// Defines the retry behavior for a failed operation. +public struct RetryPolicy: Sendable, Hashable, Codable { + /// The maximum number of times to retry the operation. A value of 0 means no retries. + public let maxRetries: Int + + /// The multiplier to apply to the delay between retries. A value of 1.0 is a linear backoff. + public let backoffMultiplier: Double + + /// The initial delay to wait before the first retry. + public let initialDelay: TimeInterval + + /// The maximum possible delay between retries. + public let maxDelay: TimeInterval + + public init( + maxRetries: Int = 3, + backoffMultiplier: Double = 2.0, + initialDelay: TimeInterval = 1.0, + maxDelay: TimeInterval = 60.0 + ) { + self.maxRetries = maxRetries + self.backoffMultiplier = backoffMultiplier + self.initialDelay = initialDelay + self.maxDelay = maxDelay + } +} + +// MARK: - Operation Extensions + +extension Operation { + /// Compute a content digest for cache key generation using stable serialization + /// - Throws: OperationError if encoding or digest computation fails + public func contentDigest() throws -> Digest { + var hasher = SHA256() + + // For now, use a simple string representation of the operation + // In production, this would need proper type-specific hashing + let operationString = String(describing: self) + guard let operationData = operationString.data(using: String.Encoding.utf8) else { + // This should never happen as String descriptions are valid UTF-8 + throw OperationError.encodingFailed("Failed to encode operation string as UTF-8: \(operationString)") + } + hasher.update(data: operationData) + + let digest = hasher.finalize() + do { + return try Digest(algorithm: .sha256, bytes: Data(digest)) + } catch { + // This should never happen as SHA256 produces correct byte length + throw OperationError.digestComputationFailed(error) + } + } +} + +/// Source location information. +public struct SourceLocation: Sendable, Hashable { + public let file: String + public let line: Int + public let column: Int? + + public init(file: String, line: Int, column: Int? = nil) { + self.file = file + self.line = line + self.column = column + } +} + +/// Cache configuration for operations. +public struct CacheConfig: Sendable, Hashable { + public enum CacheMode: String, Sendable, Hashable { + case `default` + case none + case locked + case shared + } + + public let mode: CacheMode + public let id: String? + public let sharing: SharingMode? + + public init(mode: CacheMode = .default, id: String? = nil, sharing: SharingMode? = nil) { + self.mode = mode + self.id = id + self.sharing = sharing + } +} + +/// Cache sharing mode. +public enum SharingMode: String, Sendable, Hashable { + case locked + case shared + case `private` +} + +/// Attribute value for extensible metadata. +public enum AttributeValue: Sendable, Hashable { + case string(String) + case integer(Int) + case double(Double) + case boolean(Bool) + case data(Data) + case array([AttributeValue]) + case dictionary([String: AttributeValue]) +} + +// MARK: - Codable Support + +extension OperationKind: Codable {} + +extension OperationMetadata: Codable { + // Implementation would handle encoding/decoding of all fields +} + +extension SourceLocation: Codable {} + +extension CacheConfig.CacheMode: Codable {} +extension CacheConfig: Codable {} + +extension SharingMode: Codable {} + +extension AttributeValue: Codable { + // Implementation would handle all cases +} diff --git a/Sources/NativeBuilder/ContainerBuildIR/Platform.swift b/Sources/NativeBuilder/ContainerBuildIR/Platform.swift new file mode 100644 index 00000000..82edb8d4 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/Platform.swift @@ -0,0 +1,26 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOCI +import Foundation + +extension Platform { + /// Linux on AMD64 (Intel/AMD 64-bit) + public static let linuxAMD64 = Platform(arch: "amd64", os: "linux") + + /// Linux on ARM64 (64-bit ARM) + public static let linuxARM64 = Platform(arch: "arm64", os: "linux") +} diff --git a/Sources/NativeBuilder/ContainerBuildIR/Reference.swift b/Sources/NativeBuilder/ContainerBuildIR/Reference.swift new file mode 100644 index 00000000..c8464dd1 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/Reference.swift @@ -0,0 +1,291 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// A reference to a container image. +/// +/// Design rationale: +/// - Supports all common reference formats (registry, tag, digest) +/// - Validates references on creation to catch errors early +/// - Immutable to ensure references remain valid +/// - Follows Docker/OCI reference specification +public struct ImageReference: Hashable, Sendable { + /// Registry host (e.g., "docker.io", "ghcr.io") + public let registry: String? + + /// Repository path (e.g., "library/ubuntu", "myorg/myapp") + public let repository: String + + /// Tag (e.g., "latest", "v1.2.3") + public let tag: String? + + /// Digest for content-addressed reference + public let digest: Digest? + + /// Create an image reference + /// - Note: Must have either tag or digest (or both) + public init( + registry: String? = nil, + repository: String, + tag: String? = nil, + digest: Digest? = nil + ) throws { + guard tag != nil || digest != nil else { + throw ReferenceError.missingTagOrDigest + } + + // Validate repository format + guard Self.isValidRepository(repository) else { + throw ReferenceError.invalidRepository(repository) + } + + // Validate registry if provided + if let registry = registry { + guard Self.isValidRegistry(registry) else { + throw ReferenceError.invalidRegistry(registry) + } + } + + self.registry = registry + self.repository = repository + self.tag = tag + self.digest = digest + } + + /// Parse an image reference string (e.g., "ubuntu:20.04", "ghcr.io/myorg/app@sha256:...") + public init?(parsing string: String) { + // Handle digest references (containing @) + let digestSplit = string.split(separator: "@", maxSplits: 1) + let beforeDigest = String(digestSplit[0]) + let digest: Digest? + if digestSplit.count == 2 { + let digestString = String(digestSplit[1]) + // If it starts with sha256:, it's already in the right format + // Otherwise prepend sha256: + let fullDigestString = digestString.hasPrefix("sha256:") ? digestString : "sha256:\(digestString)" + guard let d = try? Digest(parsing: fullDigestString) else { return nil } + digest = d + } else { + digest = nil + } + + // First, determine if we have a registry by looking at the first component + let pathComponents = beforeDigest.split(separator: "/") + + let registry: String? + let repoAndTag: String + + if pathComponents.count >= 2 { + let firstComponent = String(pathComponents[0]) + // Check if first component looks like a registry + // It's a registry if it contains a dot (domain) or colon (port) or is "localhost" + if firstComponent.contains(".") || firstComponent.contains(":") || firstComponent == "localhost" { + registry = firstComponent + repoAndTag = pathComponents.dropFirst().joined(separator: "/") + } else { + registry = nil + repoAndTag = beforeDigest + } + } else { + registry = nil + repoAndTag = beforeDigest + } + + // Now handle tag in the repository part + let tagSplit = repoAndTag.split(separator: ":", maxSplits: 1) + let repository = String(tagSplit[0]) + let tag: String? = tagSplit.count == 2 ? String(tagSplit[1]) : nil + + do { + try self.init( + registry: registry, + repository: repository, + tag: tag ?? (digest == nil ? "latest" : nil), + digest: digest + ) + } catch { + return nil + } + } + + /// Full reference string + public var stringValue: String { + var result = "" + + if let registry = registry { + result += registry + "/" + } + + result += repository + + if let tag = tag { + result += ":" + tag + } + + if let digest = digest { + result += "@" + digest.stringValue + } + + return result + } + + /// Reference without registry (for local use) + public var localReference: String { + var result = repository + + if let tag = tag { + result += ":" + tag + } + + if let digest = digest { + result += "@" + digest.stringValue + } + + return result + } + + // MARK: - Validation + + private static func isValidRepository(_ repo: String) -> Bool { + // Basic validation - can be enhanced + !repo.isEmpty && repo.allSatisfy { $0.isLetter || $0.isNumber || $0 == "/" || $0 == "-" || $0 == "_" || $0 == "." } + } + + private static func isValidRegistry(_ registry: String) -> Bool { + // Must contain a dot or colon (to distinguish from repository) + registry.contains(".") || registry.contains(":") + } + + private static func looksLikeRegistry(_ component: String) -> Bool { + // Contains dot (domain) or is "localhost" + // Note: Don't check for colon here as it could be a tag separator + component.contains(".") || component == "localhost" + } +} + +/// A reference to a build stage. +/// +/// Design rationale: +/// - Supports both named stages and index-based references +/// - Type-safe to prevent mixing stage and image references +/// - Lightweight for efficient graph operations +public enum StageReference: Hashable, Sendable { + /// Reference by stage name (FROM ubuntu AS builder -> "builder") + case named(String) + + /// Reference by stage index (0-based) + case index(Int) + + /// The previous stage (used for implicit references) + case previous + + public var stringValue: String { + switch self { + case .named(let name): + return name + case .index(let idx): + return String(idx) + case .previous: + return "" + } + } +} + +// MARK: - Errors + +public enum ReferenceError: LocalizedError { + case missingTagOrDigest + case invalidRepository(String) + case invalidRegistry(String) + case invalidFormat(String) + + public var errorDescription: String? { + switch self { + case .missingTagOrDigest: + return "Image reference must have either a tag or digest" + case .invalidRepository(let repo): + return "Invalid repository format: '\(repo)'" + case .invalidRegistry(let registry): + return "Invalid registry format: '\(registry)'" + case .invalidFormat(let string): + return "Invalid reference format: '\(string)'" + } + } +} + +// MARK: - Codable + +extension ImageReference: Codable { + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + let string = try container.decode(String.self) + guard let parsed = ImageReference(parsing: string) else { + throw DecodingError.dataCorruptedError( + in: container, + debugDescription: "Invalid image reference: \(string)" + ) + } + self = parsed + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(stringValue) + } +} + +extension StageReference: Codable { + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + + if let index = try? container.decode(Int.self) { + self = .index(index) + } else if let name = try? container.decode(String.self) { + if name == "" { + self = .previous + } else { + self = .named(name) + } + } else { + throw DecodingError.dataCorrupted( + DecodingError.Context(codingPath: decoder.codingPath, debugDescription: "Invalid stage reference") + ) + } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + switch self { + case .named(let name): + try container.encode(name) + case .index(let idx): + try container.encode(idx) + case .previous: + try container.encode("") + } + } +} + +// MARK: - CustomStringConvertible + +extension ImageReference: CustomStringConvertible { + public var description: String { stringValue } +} + +extension StageReference: CustomStringConvertible { + public var description: String { stringValue } +} diff --git a/Sources/NativeBuilder/ContainerBuildIR/Serialization/IRCoder.swift b/Sources/NativeBuilder/ContainerBuildIR/Serialization/IRCoder.swift new file mode 100644 index 00000000..d44325a3 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/Serialization/IRCoder.swift @@ -0,0 +1,321 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOCI +import Foundation + +/// Protocol for encoding/decoding operations. +/// +/// Design rationale: +/// - Type-erased operations need special handling +/// - Preserve unknown operation types for forward compatibility +/// - Support multiple serialization formats +public protocol IRCoder { + func encode(_ graph: BuildGraph) throws -> Data + func decode(_ data: Data) throws -> BuildGraph +} + +/// JSON-based IR coder. +/// +/// Design rationale: +/// - Human-readable for debugging +/// - Wide tooling support +/// - Good balance of size and readability +public struct JSONIRCoder: IRCoder { + private let encoder: JSONEncoder + private let decoder: JSONDecoder + + public init(prettyPrint: Bool = false) { + encoder = JSONEncoder() + decoder = JSONDecoder() + + if prettyPrint { + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + } + + // Configure date encoding + encoder.dateEncodingStrategy = .iso8601 + decoder.dateDecodingStrategy = .iso8601 + } + + public func encode(_ graph: BuildGraph) throws -> Data { + let container = try BuildGraphContainer(graph: graph) + return try encoder.encode(container) + } + + public func decode(_ data: Data) throws -> BuildGraph { + let container = try decoder.decode(BuildGraphContainer.self, from: data) + return try container.toBuildGraph() + } +} + +/// Container for serializing build graphs. +/// +/// Design rationale: +/// - Wraps the graph with version information +/// - Enables format evolution +private struct BuildGraphContainer: Codable { + let version: String + let graph: SerializedBuildGraph + + init(graph: BuildGraph) throws { + self.version = "1.0" + self.graph = try SerializedBuildGraph(from: graph) + } + + func toBuildGraph() throws -> BuildGraph { + try graph.toBuildGraph() + } +} + +/// Serializable representation of BuildGraph. +private struct SerializedBuildGraph: Codable { + let stages: [SerializedStage] + let buildArgs: [String: String] + let targetPlatforms: Set + let metadata: BuildGraphMetadata + + init(from graph: BuildGraph) throws { + self.stages = try graph.stages.map { try SerializedStage(from: $0) } + self.buildArgs = graph.buildArgs + self.targetPlatforms = graph.targetPlatforms + self.metadata = graph.metadata + } + + func toBuildGraph() throws -> BuildGraph { + let stages = try self.stages.map { try $0.toBuildStage() } + return try BuildGraph( + stages: stages, + buildArgs: buildArgs, + targetPlatforms: targetPlatforms, + metadata: metadata + ) + } +} + +/// Serializable representation of BuildStage. +private struct SerializedStage: Codable { + let id: UUID + let name: String? + let base: SerializedOperation + let nodes: [SerializedNode] + let platform: Platform? + + init(from stage: BuildStage) throws { + self.id = stage.id + self.name = stage.name + self.base = try SerializedOperation(from: stage.base) + self.nodes = try stage.nodes.map { try SerializedNode(from: $0) } + self.platform = stage.platform + } + + func toBuildStage() throws -> BuildStage { + guard let baseOp = try base.toOperation() as? ImageOperation else { + throw IRDecodingError.invalidOperationType( + expected: "ImageOperation", + actual: String(describing: type(of: base)) + ) + } + + let nodes = try self.nodes.map { try $0.toBuildNode() } + + return BuildStage( + id: id, + name: name, + base: baseOp, + nodes: nodes, + platform: platform + ) + } +} + +/// Serializable representation of BuildNode. +private struct SerializedNode: Codable { + let id: UUID + let operation: SerializedOperation + let dependencies: Set + let cacheKey: CacheKey? + let constraints: Set + + init(from node: BuildNode) throws { + self.id = node.id + self.operation = try SerializedOperation(from: node.operation) + self.dependencies = node.dependencies + self.cacheKey = node.cacheKey + self.constraints = node.constraints + } + + func toBuildNode() throws -> BuildNode { + BuildNode( + id: id, + operation: try operation.toOperation(), + dependencies: dependencies, + cacheKey: cacheKey, + constraints: constraints + ) + } +} + +/// Serializable representation of Operation. +/// +/// Design rationale: +/// - Type-erased operations need explicit type tracking +/// - Support for unknown operation types (forward compatibility) +private struct SerializedOperation: Codable { + let kind: OperationKind + let data: Data + + init(from operation: any Operation) throws { + self.kind = operation.operationKind + + // Encode the specific operation type + let encoder = JSONEncoder() + switch operation { + case let op as ExecOperation: + self.data = try encoder.encode(op) + case let op as FilesystemOperation: + self.data = try encoder.encode(op) + case let op as ImageOperation: + self.data = try encoder.encode(op) + case let op as MetadataOperation: + self.data = try encoder.encode(op) + default: + // For unknown types, try generic encoding + guard let encodable = operation as? Encodable else { + throw IREncodingError.unsupportedOperationType(kind) + } + self.data = try encoder.encode(AnyEncodable(encodable)) + } + } + + func toOperation() throws -> any Operation { + let decoder = JSONDecoder() + + switch kind { + case .exec: + return try decoder.decode(ExecOperation.self, from: data) + case .filesystem: + return try decoder.decode(FilesystemOperation.self, from: data) + case .image: + return try decoder.decode(ImageOperation.self, from: data) + case .metadata: + return try decoder.decode(MetadataOperation.self, from: data) + default: + // Unknown operation type - preserve for forward compatibility + throw IRDecodingError.unknownOperationType(kind) + } + } +} + +/// Type-erased encodable wrapper. +private struct AnyEncodable: Encodable { + private let encode: (Encoder) throws -> Void + + init(_ encodable: Encodable) { + self.encode = encodable.encode + } + + func encode(to encoder: Encoder) throws { + try encode(encoder) + } +} + +// MARK: - Binary Coder + +/// Binary IR coder for compact representation. +/// +/// Design rationale: +/// - Optimized for size and speed +/// - Uses property list binary format +/// - Good for cache storage +public struct BinaryIRCoder: IRCoder { + public init() {} + + public func encode(_ graph: BuildGraph) throws -> Data { + // First encode to intermediate format + let jsonCoder = JSONIRCoder() + let jsonData = try jsonCoder.encode(graph) + + // Convert to property list + let jsonObject = try JSONSerialization.jsonObject(with: jsonData) + return try PropertyListSerialization.data( + fromPropertyList: jsonObject, + format: .binary, + options: 0 + ) + } + + public func decode(_ data: Data) throws -> BuildGraph { + // Decode from property list + let plistObject = try PropertyListSerialization.propertyList( + from: data, + format: nil + ) + + // Convert back to JSON + let jsonData = try JSONSerialization.data(withJSONObject: plistObject) + + // Decode using JSON coder + let jsonCoder = JSONIRCoder() + return try jsonCoder.decode(jsonData) + } +} + +// MARK: - Errors + +public enum IREncodingError: LocalizedError { + case unsupportedOperationType(OperationKind) + + public var errorDescription: String? { + switch self { + case .unsupportedOperationType(let kind): + return "Cannot encode operation type: \(kind.rawValue)" + } + } +} + +public enum IRDecodingError: LocalizedError { + case unknownOperationType(OperationKind) + case invalidOperationType(expected: String, actual: String) + case invalidFormat + + public var errorDescription: String? { + switch self { + case .unknownOperationType(let kind): + return "Unknown operation type: \(kind.rawValue)" + case .invalidOperationType(let expected, let actual): + return "Expected \(expected) but got \(actual)" + case .invalidFormat: + return "Invalid IR format" + } + } +} + +// MARK: - Convenience Extensions + +extension BuildGraph { + /// Save graph to file. + public func save(to url: URL, using coder: IRCoder = JSONIRCoder(prettyPrint: true)) throws { + let data = try coder.encode(self) + try data.write(to: url) + } + + /// Load graph from file. + public static func load(from url: URL, using coder: IRCoder = JSONIRCoder()) throws -> BuildGraph { + let data = try Data(contentsOf: url) + return try coder.decode(data) + } +} diff --git a/Sources/NativeBuilder/ContainerBuildIR/Validation/Validator.swift b/Sources/NativeBuilder/ContainerBuildIR/Validation/Validator.swift new file mode 100644 index 00000000..53350426 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildIR/Validation/Validator.swift @@ -0,0 +1,411 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// Protocol for build graph validators. +/// +/// Design rationale: +/// - Composable validation rules +/// - Collect all errors rather than fail-fast +/// - Extensible for custom validation +public protocol BuildValidator { + /// Validate a build graph + func validate(_ graph: BuildGraph) -> ValidationResult +} + +/// Result of validation. +public struct ValidationResult { + public let errors: [ValidationError] + public let warnings: [ValidationWarning] + + public var isValid: Bool { errors.isEmpty } + + public init(errors: [ValidationError] = [], warnings: [ValidationWarning] = []) { + self.errors = errors + self.warnings = warnings + } + + /// Combine multiple results + public static func combine(_ results: [ValidationResult]) -> ValidationResult { + ValidationResult( + errors: results.flatMap { $0.errors }, + warnings: results.flatMap { $0.warnings } + ) + } +} + +// MARK: - Validation Error Enum + +/// A structured error that prevents a build from proceeding. +public enum ValidationError: Error, LocalizedError, Sendable { + // Structural Errors + case duplicateNodeID(id: UUID, location: ValidationLocation) + case cyclicDependency(location: ValidationLocation) + case missingDependency(dependencyID: UUID, location: ValidationLocation) + + // Reference Errors + case undefinedStageReference(name: String, location: ValidationLocation) + case stageIndexOutOfBounds(index: Int, location: ValidationLocation) + case invalidPreviousReference(location: ValidationLocation) + + // Path Errors + case emptyDestinationPath(location: ValidationLocation) + case absoluteContextPath(path: String, location: ValidationLocation) + case emptyMountTarget(location: ValidationLocation) + + public var errorDescription: String? { + switch self { + case .duplicateNodeID(let id, _): + return "Duplicate node ID found: \(id)." + case .cyclicDependency: + return "Stage contains a cyclic dependency." + case .missingDependency(let dependencyID, _): + return "Node references a non-existent dependency: \(dependencyID)." + case .undefinedStageReference(let name, _): + return "Reference to an undefined stage: '\(name)'." + case .stageIndexOutOfBounds(let index, _): + return "Stage index is out of bounds: \(index)." + case .invalidPreviousReference: + return "Cannot reference the previous stage from the first stage." + case .emptyDestinationPath: + return "Filesystem operation has an empty destination path." + case .absoluteContextPath(let path, _): + return "Source path for a context operation must be relative, but found absolute path: '\(path)'." + case .emptyMountTarget: + return "An execution mount has an empty target path." + } + } +} + +// MARK: - Validation Warning Enum + +/// A structured warning that does not prevent a build but indicates a potential issue. +public enum ValidationWarning: Sendable { + // Reference Warnings + case forwardStageReferenceByName(name: String, location: ValidationLocation) + case forwardStageReferenceByIndex(index: Int, location: ValidationLocation) + + // Path Warnings + case pathContainsDotDot(path: String, location: ValidationLocation) + + // Security Warnings + case privilegedExecution(location: ValidationLocation) + case runningAsRoot(location: ValidationLocation) + case readWriteSecretMount(location: ValidationLocation) + + // Best Practice Warnings + case aptGetUpdateWithoutInstall(location: ValidationLocation) + case missingHealthcheck(location: ValidationLocation) + + /// A human-readable description of the warning. + public var message: String { + switch self { + case .forwardStageReferenceByName(let name, _): + return "Forward reference to stage '\(name)'. Build may be inefficient." + case .forwardStageReferenceByIndex(let index, _): + return "Forward reference to stage at index \(index). Build may be inefficient." + case .pathContainsDotDot(let path, _): + return "Path contains '..', which could lead to accessing files outside the build context: '\(path)'." + case .privilegedExecution: + return "Operation is configured to run with privileged access." + case .runningAsRoot: + return "Operation is configured to run as the root user." + case .readWriteSecretMount: + return "A secret is mounted as read-write, which is insecure." + case .aptGetUpdateWithoutInstall: + return "'apt-get update' is run in a separate command from 'apt-get install'." + case .missingHealthcheck: + return "The final image has no HEALTHCHECK defined." + } + } + + /// A suggestion for how to resolve the warning. + public var suggestion: String? { + switch self { + case .forwardStageReferenceByName, .forwardStageReferenceByIndex: + return "Consider reordering stages to ensure all dependencies are built first." + case .pathContainsDotDot: + return "Use explicit paths from the context root instead of relative parent paths." + case .privilegedExecution: + return "Ensure the operation truly requires privileged mode to run." + case .runningAsRoot: + return "Consider specifying a non-root user with the USER instruction for enhanced security." + case .readWriteSecretMount: + return "Secrets should always be mounted as read-only." + case .aptGetUpdateWithoutInstall: + return "Combine 'apt-get update' and 'apt-get install' in the same RUN command to reduce image layers and ensure cache correctness." + case .missingHealthcheck: + return "Consider adding a HEALTHCHECK instruction to your final stage for production-ready images." + } + } +} + +/// Location information for validation messages. +public enum ValidationLocation: Sendable { + case stage(name: String?) + case node(stageIndex: Int, nodeIndex: Int) + case operation(OperationKind) + case sourceLocation(SourceLocation) +} + +/// Composite validator that runs multiple validators. +public struct CompositeValidator: BuildValidator, Sendable { + private let validators: [any BuildValidator & Sendable] + + public init(validators: [any BuildValidator & Sendable]) { + self.validators = validators + } + + public func validate(_ graph: BuildGraph) -> ValidationResult { + ValidationResult.combine(validators.map { $0.validate(graph) }) + } +} + +/// Standard validator with all built-in rules. +public struct StandardValidator: BuildValidator, Sendable { + private let validator: CompositeValidator + + public init() { + validator = CompositeValidator(validators: [ + StructuralValidator(), + ReferenceValidator(), + PathValidator(), + SecurityValidator(), + BestPracticesValidator(), + ]) + } + + public func validate(_ graph: BuildGraph) -> ValidationResult { + validator.validate(graph) + } +} + +/// Validates graph structure (cycles, dependencies). +public struct StructuralValidator: BuildValidator, Sendable { + public func validate(_ graph: BuildGraph) -> ValidationResult { + var errors: [ValidationError] = [] + + // Check each stage + for (_, stage) in graph.stages.enumerated() { + let stageLocation = ValidationLocation.stage(name: stage.name) + // Check for duplicate node IDs + var seenIds = Set() + for node in stage.nodes { + if !seenIds.insert(node.id).inserted { + errors.append(.duplicateNodeID(id: node.id, location: stageLocation)) + } + } + + // Cycle detection will be done globally due to cross-stage dependencies + + // Dependencies will be checked globally after collecting all node IDs + } + + // Collect all node IDs across all stages + var allNodeIds = Set() + for stage in graph.stages { + for node in stage.nodes { + allNodeIds.insert(node.id) + } + } + + // Now check that all dependencies exist (can be cross-stage) + for (stageIndex, stage) in graph.stages.enumerated() { + for (nodeIndex, node) in stage.nodes.enumerated() { + for dep in node.dependencies { + if !allNodeIds.contains(dep) { + let nodeLocation = ValidationLocation.node(stageIndex: stageIndex, nodeIndex: nodeIndex) + errors.append(.missingDependency(dependencyID: dep, location: nodeLocation)) + } + } + } + } + + return ValidationResult(errors: errors) + } +} + +/// Validates cross-stage references. +public struct ReferenceValidator: BuildValidator, Sendable { + public func validate(_ graph: BuildGraph) -> ValidationResult { + var errors: [ValidationError] = [] + var warnings: [ValidationWarning] = [] + + for (stageIndex, stage) in graph.stages.enumerated() { + let stageLocation = ValidationLocation.stage(name: stage.name) + let stageDeps = stage.stageDependencies() + + for dep in stageDeps { + // Validate reference exists + let exists: Bool + switch dep { + case .named(let name): + exists = graph.stages.contains { $0.name == name } + if !exists { + errors.append(.undefinedStageReference(name: name, location: stageLocation)) + } + case .index(let idx): + exists = idx >= 0 && idx < graph.stages.count + if !exists { + errors.append(.stageIndexOutOfBounds(index: idx, location: stageLocation)) + } + case .previous: + exists = stageIndex > 0 + if !exists { + errors.append(.invalidPreviousReference(location: stageLocation)) + } + } + + // Check for forward references (warning) + if exists { + switch dep { + case .named(let name): + if let depIndex = graph.stages.firstIndex(where: { $0.name == name }), + depIndex > stageIndex + { + warnings.append(.forwardStageReferenceByName(name: name, location: stageLocation)) + } + case .index(let idx): + if idx > stageIndex { + warnings.append(.forwardStageReferenceByIndex(index: idx, location: stageLocation)) + } + case .previous: + break // Always valid + } + } + } + } + + return ValidationResult(errors: errors, warnings: warnings) + } +} + +/// Validates filesystem paths and operations. +public struct PathValidator: BuildValidator, Sendable { + public func validate(_ graph: BuildGraph) -> ValidationResult { + var errors: [ValidationError] = [] + var warnings: [ValidationWarning] = [] + + for stage in graph.stages { + for node in stage.nodes { + if let fsOp = node.operation as? FilesystemOperation { + let opLocation = ValidationLocation.operation(node.operation.operationKind) + // Validate destination path + if fsOp.destination.isEmpty { + errors.append(.emptyDestinationPath(location: opLocation)) + } + + // Check for absolute paths in context source + if case .context(let source) = fsOp.source { + for path in source.paths { + if path.hasPrefix("/") { + errors.append(.absoluteContextPath(path: path, location: opLocation)) + } + + if path.contains("..") { + warnings.append(.pathContainsDotDot(path: path, location: opLocation)) + } + } + } + } + + // Validate mount paths + if let execOp = node.operation as? ExecOperation { + let opLocation = ValidationLocation.operation(node.operation.operationKind) + for mount in execOp.mounts { + if mount.target == nil && mount.envTarget == nil { + errors.append(.emptyMountTarget(location: opLocation)) + } + } + } + } + } + + return ValidationResult(errors: errors, warnings: warnings) + } +} + +/// Validates security constraints. +public struct SecurityValidator: BuildValidator, Sendable { + public func validate(_ graph: BuildGraph) -> ValidationResult { + var warnings: [ValidationWarning] = [] + + for stage in graph.stages { + for node in stage.nodes { + if let execOp = node.operation as? ExecOperation { + let opLocation = ValidationLocation.operation(node.operation.operationKind) + // Warn about privileged execution + if execOp.security.privileged { + warnings.append(.privilegedExecution(location: opLocation)) + } + + // Warn about running as root + if execOp.user == nil { + warnings.append(.runningAsRoot(location: opLocation)) + } + + // Check for secret mounts + for mount in execOp.mounts { + if mount.type == .secret && !mount.options.readOnly { + warnings.append(.readWriteSecretMount(location: opLocation)) + } + } + } + } + } + + return ValidationResult(warnings: warnings) + } +} + +/// Validates against best practices. +public struct BestPracticesValidator: BuildValidator, Sendable { + public func validate(_ graph: BuildGraph) -> ValidationResult { + var warnings: [ValidationWarning] = [] + + for stage in graph.stages { + var hasHealthcheck = false + + for node in stage.nodes { + let opLocation = ValidationLocation.operation(node.operation.operationKind) + // Check for multiple RUN commands that could be combined + if let execOp = node.operation as? ExecOperation { + if case .shell(let cmd) = execOp.command { + if cmd.contains("apt-get update") && !cmd.contains("apt-get install") { + warnings.append(.aptGetUpdateWithoutInstall(location: opLocation)) + } + } + } + + // Track user changes + if let metaOp = node.operation as? MetadataOperation { + if case .setHealthcheck = metaOp.action { + hasHealthcheck = true + } + } + } + + // Warn if no healthcheck defined + if !hasHealthcheck && stage == graph.targetStage { + let stageLocation = ValidationLocation.stage(name: stage.name) + warnings.append(.missingHealthcheck(location: stageLocation)) + } + } + + return ValidationResult(warnings: warnings) + } +} diff --git a/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerInstructionVisitor.swift b/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerInstructionVisitor.swift new file mode 100644 index 00000000..6546cf62 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerInstructionVisitor.swift @@ -0,0 +1,139 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR + +protocol InstructionVisitor { + func visit(_ from: FromInstruction) throws + func visit(_ run: RunInstruction) throws + func visit(_ copy: CopyInstruction) throws +} + +/// DockerInstructionVisitor visits each provided DockerInstruction and builds a +/// build graph from the instructions. +public class DockerInstructionVisitor: InstructionVisitor { + + internal let graphBuilder: GraphBuilder + + init() { + self.graphBuilder = GraphBuilder() + } + + func buildGraph(from: [DockerInstruction]) throws -> BuildGraph { + for instruction in from { + try instruction.accept(self) + } + return try graphBuilder.build() + } +} + +extension DockerInstructionVisitor { + func visit(_ from: FromInstruction) throws { + if let stageName = from.stageName { + try graphBuilder.stage(name: stageName, from: from.image, platform: from.platform) + } else { + try graphBuilder.stage(from: from.image, platform: from.platform) + } + } + + func visit(_ run: RunInstruction) throws { + var mounts: [Mount] = [] + for m in run.mounts { + guard let type = m.type else { + throw ParseError.unexpectedValue + } + + let mountSource: MountSource? + switch m.type { + case .bind, .cache: + guard let source = m.source else { + throw ParseError.missingRequiredField(MountOptionNames.source.rawValue) + } + if let from = m.from, from != "" { + if let _ = graphBuilder.getStage(stageName: from) { + mountSource = .stage(.named(from), path: source) + } else if let context = graphBuilder.getBuildArg(key: from) { + mountSource = .context(context, path: source) + } else { + // mount source is an image name + guard let imageRef = ImageReference(parsing: from) else { + throw ParseError.invalidImage(from) + } + mountSource = .image(imageRef, path: source) + } + } else { + // from was not set or is empty, default is local source + mountSource = .local(source) + } + case .secret: + mountSource = .secret(m.id!) + case .ssh: + mountSource = .sshAgent + default: + // this covers .tmpfs case as well + mountSource = nil + } + + guard let options = m.options else { + throw ParseError.unexpectedValue + } + + guard let readonly = options.readonly else { + throw ParseError.unexpectedValue + } + + let mountOptions = MountOptions( + readOnly: readonly, + uid: options.uid, + gid: options.gid, + mode: options.mode, + size: options.size, + sharing: options.sharing, + required: options.required) + + let graphMount = Mount( + type: type, + target: m.target, + envTarget: m.env, + source: mountSource, + options: mountOptions) + + mounts.append(graphMount) + } + + try graphBuilder.run(run.command, shell: run.shell, mounts: mounts) + } + + func visit(_ copy: CopyInstruction) throws { + // TODO katiewasnothere: plumb through "--link" option + if let from = copy.from { + var source: FilesystemSource + if let _ = graphBuilder.getStage(stageName: from) { + source = .stage(.named(from), paths: copy.sources) + } else if let context = graphBuilder.getBuildArg(key: from) { + source = .context(ContextSource(name: context, paths: copy.sources)) + } else { + guard let imageRef = ImageReference(parsing: from) else { + throw ParseError.invalidImage(from) + } + source = .image(imageRef, paths: copy.sources) + } + try graphBuilder.copy(from: source, to: copy.destination, chown: copy.chown, chmod: copy.chmod) + return + } + try graphBuilder.copyFromContext(paths: copy.sources, to: copy.destination, chown: copy.chown, chmod: copy.chmod) + } +} diff --git a/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerfileParser.swift b/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerfileParser.swift new file mode 100644 index 00000000..09239fe7 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerfileParser.swift @@ -0,0 +1,253 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import Foundation + +/// DockerfileParser parses a dockerfile to a BuildGraph. +public struct DockerfileParser: BuildParser { + public func parse(_ input: String) throws -> BuildGraph { + var instructions = [DockerInstruction]() + let lines = input.components(separatedBy: .newlines) + var lineIndex = 0 + while lineIndex < lines.count { + var line = lines[lineIndex].trimmingCharacters(in: .whitespacesAndNewlines) + if line.isEmpty { + lineIndex += 1 + continue + } + + while lineIndex < lines.count && line.hasSuffix("\\") { + line = String(line.dropLast("\\".count)) + let next = lineIndex + 1 + if next < lines.count { + let nextLine = String(lines[next].trimmingCharacters(in: .whitespacesAndNewlines)) + line.append(nextLine) + lineIndex += 1 + } + } + + var tokenizer = DockerfileTokenizer(line) + let tokens = try tokenizer.getTokens() + + try instructions.append(tokensToDockerInstruction(tokens: tokens)) + + lineIndex += 1 + } + let visitor = DockerInstructionVisitor() + return try visitor.buildGraph(from: instructions) + } + + private func tokensToDockerInstruction(tokens: [Token]) throws -> DockerInstruction { + guard case .stringLiteral(let value) = tokens.first else { + throw ParseError.missingInstruction + } + + let instruction = DockerInstructionName(rawValue: value.lowercased()) + + switch instruction { + case .FROM: + return try tokensToFromInstruction(tokens: tokens) + case .RUN: + return try tokensToRunInstruction(tokens: tokens) + default: + throw ParseError.invalidInstruction(value) + } + } + + internal func tokensToFromInstruction(tokens: [Token]) throws -> FromInstruction { + var index = tokens.startIndex + index += 1 // skip the instruction + + var stageName: String? + var platform: String? + var imageName: String? + + // Step 1: parse options + while index < tokens.endIndex { + guard case .option(let key, let value) = tokens[index] else { + break + } + guard FromOptions(rawValue: key) == .platform else { + throw ParseError.unexpectedValue + } + platform = value + index += 1 + } + + // Step 2: Parse image name + if index < tokens.endIndex { + guard case .stringLiteral(let value) = tokens[index] else { + throw ParseError.unexpectedValue + } + imageName = value + index += 1 + } + + // Step 3 (optional): Parse stage name + if index < tokens.endIndex { + guard case .stringLiteral(let value) = tokens[index], + DockerKeyword(rawValue: value.lowercased()) == .AS + else { + throw ParseError.unexpectedValue + } + index += 1 + guard index < tokens.endIndex, case .stringLiteral(let name) = tokens[index] else { + throw ParseError.invalidSyntax + } + stageName = name + index += 1 + } + + guard let imageName = imageName else { + throw ParseError.invalidSyntax + } + + // check for extra tokens + if index < tokens.endIndex { + throw ParseError.unexpectedValue + } + + return try FromInstruction(image: imageName, platform: platform, stageName: stageName) + } + + internal func tokensToRunInstruction(tokens: [Token]) throws -> RunInstruction { + var index = tokens.startIndex + index += 1 // skip the instruction + + var rawMounts = [String]() + var network: String? = nil + + // Step 1: parse options + while index < tokens.endIndex { + guard case .option(let key, let value) = tokens[index] else { + break + } + + guard let option = RunOptions(rawValue: key) else { + throw ParseError.unexpectedValue + } + + switch option { + case .mount: + rawMounts.append(value) + case .network: + network = value + default: + throw ParseError.unexpectedValue + } + index += 1 + } + + var command = [String]() + var shell = true + + // Step 2: parse run command and if we're using shell or exec form + while index < tokens.endIndex { + if case .stringList(let value) = tokens[index], command.isEmpty { + // when using the exec form, there should only be a single list for the command + // if there's other content in the command already, the input was invalid + command = value + shell = false + index += 1 + break + } else if case .stringLiteral(let value) = tokens[index] { + command.append(value) + } else { + break + } + index += 1 + } + + // check for extra tokens + if index < tokens.endIndex { + throw ParseError.unexpectedValue + } + + return try RunInstruction(command: command, shell: shell, rawMounts: rawMounts, network: network) + } + + internal func tokensToCopyInstruction(tokens: [Token]) throws -> CopyInstruction { + var index = tokens.startIndex + index += 1 // skip the instruction + + var from: String? = nil + var chmod: String? = nil + var chown: String? = nil + var link: String? = nil + + // Step 1: parse options + while index < tokens.endIndex { + guard case .option(let key, let value) = tokens[index] else { + break + } + + guard let option = CopyOptions(rawValue: key) else { + throw ParseError.unexpectedValue + } + + switch option { + case .from: + if from != nil { + throw ParseError.duplicateOptionSet(CopyOptions.from.rawValue) + } + from = value + case .chown: + if chown != nil { + throw ParseError.duplicateOptionSet(CopyOptions.chown.rawValue) + } + chown = value + case .chmod: + if chmod != nil { + throw ParseError.duplicateOptionSet(CopyOptions.chmod.rawValue) + } + chmod = value + case .link: + if link != nil { + throw ParseError.duplicateOptionSet(CopyOptions.link.rawValue) + } + link = value + default: + throw ParseError.unexpectedValue + } + index += 1 + } + + // Step 2: Get all source paths and destination path + var sources: [String] = [] + var destination: String? + while index < tokens.endIndex { + guard case .stringLiteral(let value) = tokens[index] else { + break + } + if index + 1 == tokens.endIndex { + // this is the last path provided, it must be the destination + destination = value + } else { + sources.append(value) + } + index += 1 + } + + // check for extra tokens + if index < tokens.endIndex { + throw ParseError.unexpectedValue + } + + return try CopyInstruction(sources: sources, destination: destination, from: from, ownership: chown, permissions: chmod) + } + +} diff --git a/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerfileTokenizer.swift b/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerfileTokenizer.swift new file mode 100644 index 00000000..36e154cd --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerfileTokenizer.swift @@ -0,0 +1,144 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// DockerfileTokenizer takes as input a line from a dockerfile and outputs an array +/// of Tokens that represent the line's contents +struct DockerfileTokenizer { + private let input: String + private var position: String.Index + private let endPosition: String.Index + + public init(_ from: String) { + input = from + position = input.startIndex + endPosition = input.endIndex + } + + mutating func getTokens() throws -> [Token] { + var results = [Token]() + + while position < endPosition { + let char = input[position] + if char.isWhitespace { + // ignore white spaces that are not part of other things + position = input.index(after: position) + continue + } + + if char == "\"" || char == "'" { + position = input.index(after: position) // do not include the initial quote + let start = position + parseQuotedString() + + let quote = String(input[start.. Token { + let start = position + while position < endPosition { + let char = input[position] + if char == "]" { + // we want to include the ending ] in the rawJSON so that swift can + // correctly handle decoding the value + position = input.index(after: position) + break + } + position = input.index(after: position) + } + let rawJSON = String(input[start.. Token { + let wordStart = position + parseWord() + + let rawWord = input[wordStart.. Ownership? { + guard let input = input, !input.isEmpty else { + return Ownership(user: .numeric(id: 0), group: .numeric(id: 0)) + } + var user: OwnershipID? = nil + var group: OwnershipID? = nil + + let components = input.components(separatedBy: ":") + guard components.count <= 2 else { + throw ParseError.invalidOption(input) + } + user = parseID(id: components[0]) + if components.count == 2 { + group = parseID(id: components[1]) + } + if user == nil && group == nil { + throw ParseError.invalidOption(input) + } + return Ownership(user: user, group: group) + } + + static private func parseID(id: String) -> OwnershipID? { + if id == "" { + return nil + } + if let numberID = UInt32(id) { + return .numeric(id: numberID) + } + return .named(id: id) + } + + static internal func parsePermissions(input: String?) throws -> Permissions? { + guard let input = input else { + return nil + } + guard let mode = UInt32(input) else { + throw ParseError.invalidUint32Option(input) + } + return Permissions.mode(mode) + } + + func accept(_ visitor: DockerInstructionVisitor) throws { + try visitor.visit(self) + } +} diff --git a/Sources/NativeBuilder/ContainerBuildParser/Docker/Instructions/DockerInstruction.swift b/Sources/NativeBuilder/ContainerBuildParser/Docker/Instructions/DockerInstruction.swift new file mode 100644 index 00000000..5d536e6c --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildParser/Docker/Instructions/DockerInstruction.swift @@ -0,0 +1,64 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerizationOCI + +/// DockerInstruction represents a single docker instruction with its given options +/// and arguments. Instructions are "visited" to add to a build graph. +protocol DockerInstruction { + func accept(_ visitor: DockerInstructionVisitor) throws +} + +enum FromOptions: String { + case platform = "--platform" +} + +struct FromInstruction: DockerInstruction, Equatable { + let image: ImageReference + let platform: Platform? + let stageName: String? + + init(image: String, platform: String? = nil, stageName: String? = nil) throws { + guard let imageRef = ImageReference(parsing: image) else { + throw ParseError.invalidImage(image) + } + + var platformSpec = Platform.current + if let platform = platform { + platformSpec = try Platform(from: platform) + } + self.image = imageRef + self.platform = platformSpec + self.stageName = stageName + } + + func accept(_ visitor: DockerInstructionVisitor) throws { + try visitor.visit(self) + } +} + +/// DockerInstructionName defines a dockerfile instruction such as FROM, RUN, etc. +enum DockerInstructionName: String { + case FROM = "from" + case RUN = "run" +} + +/// DockerKeyword defines words that are used as keywords within a line of a dockerfile +/// to provide additional instruction +enum DockerKeyword: String { + case AS = "as" +} diff --git a/Sources/NativeBuilder/ContainerBuildParser/Docker/Instructions/RunInstruction.swift b/Sources/NativeBuilder/ContainerBuildParser/Docker/Instructions/RunInstruction.swift new file mode 100644 index 00000000..4047b1e0 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildParser/Docker/Instructions/RunInstruction.swift @@ -0,0 +1,392 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import Foundation + +enum RunOptions: String { + case mount = "--mount" + case network = "--network" +} + +enum MountOptionNames: String { + case type = "type" + case source = "source" + case from = "from" + case target = "target" + case dst = "dst" + case destination = "destination" + + // permissions related + case readonly = "readonly" + case ro = "ro" + case readwrite = "readwrite" + case rw = "rw" + case uid = "uid" + case gid = "gid" + case mode = "mode" + case size = "size" + + case sharing = "sharing" + case required = "required" + case env = "env" + case id = "id" +} + +extension MountType { + var allowedOptions: Set { + switch self { + case .bind: + return [.source, .from, .target, .destination, .dst, .readwrite, .rw] + case .cache: + return [.id, .target, .destination, .dst, .readonly, .ro, .sharing, .from, .source, .mode, .gid, .uid] + case .tmpfs: + return [.target, .dst, .destination, .size] + case .secret: + return [.id, .target, .dst, .destination, .env, .required, .mode, .uid, .gid] + case .ssh: + return [.id, .target, .dst, .destination, .required, .mode, .uid, .gid] + } + } +} + +/// RunMount represents a mount option with its suboptions from a docker RUN instruction +struct RunMount: Equatable { + var type: MountType? + var source: String? + var from: String? + var id: String? + var env: String? + var target: String? + var options: RunMountOptions? + + init() {} + + init( + type: MountType? = nil, + source: String? = nil, + from: String? = nil, + id: String? = nil, + env: String? = nil, + target: String? = nil, + options: RunMountOptions? = nil + ) { + self.type = type + self.source = source + self.from = from + self.id = id + self.env = env + self.target = target + self.options = options + } + + mutating internal func setOption(_ keyPath: WritableKeyPath, _ value: T) throws { + guard self.options?[keyPath: keyPath] == nil else { + throw ParseError.invalidOption("\(keyPath):\(value)") + } + if self.options == nil { + self.options = RunMountOptions() + } + if self.options?[keyPath: keyPath] != nil { + throw ParseError.duplicateOptionSet("\(keyPath):\(value)") + } + self.options?[keyPath: keyPath] = value + } + + mutating internal func setField(_ keyPath: WritableKeyPath, _ value: T) throws { + if self[keyPath: keyPath] != nil { + throw ParseError.invalidOption("\(keyPath)") + } + self[keyPath: keyPath] = value + } + + /// Validate required fields are set and set any defaults iff they are not already set + mutating internal func finalize() throws { + switch self.type { + case .bind: + if self.target == nil { + throw ParseError.missingRequiredField(MountOptionNames.target.rawValue) + } + if self.source == nil { + try self.setField(\.source, "/") + } + if self.options?.readonly == nil { + try self.setOption(\.readonly, true) + } + case .cache: + if self.target == nil { + throw ParseError.missingRequiredField(MountOptionNames.target.rawValue) + } + if self.id == nil { + try self.setField(\.id, self.target!) + } + if self.options?.readonly == nil { + try self.setOption(\.readonly, false) + } + if self.options?.sharing == nil { + try self.setOption(\.sharing, .shared) + } + if self.from == nil { + try self.setField(\.from, "") + } + if self.source == nil { + try self.setField(\.source, "/") + } + if self.options?.mode == nil { + try self.setOption(\.mode, 0755) + } + if self.options?.uid == nil { + try self.setOption(\.uid, 0) + } + if self.options?.gid == nil { + try self.setOption(\.gid, 0) + } + case .tmpfs: + if self.target == nil { + throw ParseError.missingRequiredField(MountOptionNames.target.rawValue) + } + if self.options?.readonly == nil { + try self.setOption(\.readonly, false) + } + case .secret: + if self.target == nil { + if self.env == nil { + guard let id = self.id else { + throw ParseError.missingRequiredField("id must be set when target and env are unset") + } + try self.setField(\.target, "/run/secrets/\(id)") + } + } + if id == nil { + guard let target = self.target else { + throw ParseError.missingRequiredField("target must be set when id is unset") + } + let targetURL = URL(string: target) + guard let targetURL = targetURL else { + throw ParseError.invalidOption("target is not a valid url \(target)") + } + try self.setField(\.id, targetURL.lastPathComponent) + } + if self.options?.readonly == nil { + try self.setOption(\.readonly, true) + } + if self.options?.required == nil { + try self.setOption(\.required, false) + } + if self.options?.mode == nil { + try self.setOption(\.mode, 0400) + } + if self.options?.uid == nil { + try self.setOption(\.uid, 0) + } + if self.options?.gid == nil { + try self.setOption(\.gid, 0) + } + case .ssh: + if self.id == nil { + try self.setField(\.id, "default") + } + if self.target == nil { + // TODO katiewasnothere add sufix based on number of agents added + try self.setField(\.target, "/run/buildkit/ssh_agent") + } + if self.options?.readonly == nil { + try self.setOption(\.readonly, true) + } + if self.options?.required == nil { + try self.setOption(\.required, false) + } + if self.options?.mode == nil { + try self.setOption(\.mode, 0600) + } + if self.options?.uid == nil { + try self.setOption(\.uid, 0) + } + if self.options?.gid == nil { + try self.setOption(\.gid, 0) + } + default: + throw ParseError.invalidOption("unsupported mount type \(String(describing: self.type))") + } + } +} + +/// RunMountOptions represent the suboptions set on a RUN mount option +struct RunMountOptions: Equatable { + var readonly: Bool? + var required: Bool? + var uid: UInt32? + var gid: UInt32? + var mode: UInt32? + var size: UInt32? + var sharing: SharingMode? + + init() {} + + init( + readonly: Bool? = nil, + required: Bool? = nil, + uid: UInt32? = nil, + gid: UInt32? = nil, + mode: UInt32? = nil, + size: UInt32? = nil, + sharing: SharingMode? = nil + ) { + self.readonly = readonly + self.required = required + self.uid = uid + self.gid = gid + self.mode = mode + self.size = size + self.sharing = sharing + } +} + +/// RunInstruction represents a RUN instruction from a dockerfile +struct RunInstruction: DockerInstruction, Equatable { + let command: String + let shell: Bool + let mounts: [RunMount] + let network: NetworkMode + + init() { + self.command = "" + self.shell = false + self.mounts = [] + self.network = .default + } + + init(command: [String], shell: Bool, rawMounts: [String], network: String?) throws { + self.command = command.joined(separator: " ") + self.shell = shell + self.network = try RunInstruction.parseNetworkMode(mode: network) + var parsedMounts: [RunMount] = [] + for m in rawMounts { + parsedMounts.append(try RunInstruction.parseMount(m)) + } + self.mounts = parsedMounts + } + + static internal func parseNetworkMode(mode: String?) throws -> NetworkMode { + guard let mode = mode else { + return .default + } + guard let nMode = NetworkMode(rawValue: mode) else { + throw ParseError.invalidOption(mode) + } + return nMode + } + + static internal func parseMount(_ rawMount: String) throws -> RunMount { + let components = rawMount.components(separatedBy: ",") + if components.isEmpty { + throw ParseError.invalidOption("no options set on mount") + } + + var runMount = RunMount() + for c in components { + let optionComps = c.components(separatedBy: "=") + guard optionComps.count == 2 else { + throw ParseError.invalidOption("option \(c) is not in the form key=value") + } + guard optionComps[1] != "" else { + throw ParseError.invalidOption("option \(c) is not in the form key=value") + } + let key = optionComps[0] + let value = optionComps[1] + guard let mountOption = MountOptionNames(rawValue: key) else { + throw ParseError.invalidOption("option \(key) is not supported") + } + + if let type = runMount.type { + guard type.allowedOptions.contains(mountOption) else { + throw ParseError.invalidOption("option \(mountOption) is not supported for type \(type)") + } + } else { + if let mountType = MountType(rawValue: value) { + runMount.type = mountType + continue + } else { + // still need to eval this option, so we need to go to the switch + // statement from here + runMount.type = .bind + } + } + + switch mountOption { + case .id: + try runMount.setField(\.id, value) + case .env: + try runMount.setField(\.env, value) + case .source: + try runMount.setField(\.source, value) + case .from: + try runMount.setField(\.from, value) + case .dst, .target, .destination: + try runMount.setField(\.target, value) + case .readonly, .ro: + guard let readonly = Bool(value) else { + throw ParseError.invalidBoolOption(value) + } + try runMount.setOption(\.readonly, readonly) + case .readwrite, .rw: + guard let readwrite = Bool(value) else { + throw ParseError.invalidBoolOption(value) + } + try runMount.setOption(\.readonly, !readwrite) + case .gid: + guard let gid = UInt32(value) else { + throw ParseError.invalidUint32Option(value) + } + try runMount.setOption(\.gid, gid) + case .uid: + guard let uid = UInt32(value) else { + throw ParseError.invalidUint32Option(value) + } + try runMount.setOption(\.uid, uid) + case .mode: + guard let mode = UInt32(value) else { + throw ParseError.invalidUint32Option(value) + } + try runMount.setOption(\.mode, mode) + case .size: + guard let size = UInt32(value) else { + throw ParseError.invalidUint32Option(value) + } + try runMount.setOption(\.size, size) + case .sharing: + guard let sharing = SharingMode(rawValue: value) else { + throw ParseError.invalidOption("invalid sharing type \(value)") + } + try runMount.setOption(\.sharing, sharing) + case .required: + guard let requiredVal = Bool(value) else { + throw ParseError.invalidBoolOption(value) + } + try runMount.setOption(\.required, requiredVal) + default: + throw ParseError.invalidOption("\(key) unsupported") + } + } + + try runMount.finalize() + return runMount + } + + func accept(_ visitor: DockerInstructionVisitor) throws { + try visitor.visit(self) + } +} diff --git a/Sources/NativeBuilder/ContainerBuildParser/Types.swift b/Sources/NativeBuilder/ContainerBuildParser/Types.swift new file mode 100644 index 00000000..48efb1cc --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildParser/Types.swift @@ -0,0 +1,45 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR + +public protocol BuildParser { + associatedtype Input + func parse(_ input: Input) throws -> BuildGraph +} + +/// Error types encountered while parsing. +/// TODO: These will be removed/enhanced +public enum ParseError: Error, Equatable { + case invalidImage(String) + case missingInstruction + case invalidInstruction(String) + case unexpectedValue + case invalidOption(String) + case missingRequiredField(String) + case duplicateOptionSet(String) + case invalidSyntax + case invalidBoolOption(String) + case invalidUint32Option(String) +} + +/// Token represents a logical unit within a line of builder input, such as +/// a dockerfile +public enum Token: Sendable, Equatable { + case stringLiteral(String) + case stringList([String]) + case option(String, String) +} diff --git a/Sources/NativeBuilder/ContainerBuildReporting/BaseProgressConsumer.swift b/Sources/NativeBuilder/ContainerBuildReporting/BaseProgressConsumer.swift new file mode 100644 index 00000000..64e5b318 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildReporting/BaseProgressConsumer.swift @@ -0,0 +1,179 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// Base implementation for progress consumers that provides common statistics tracking. +/// +/// This class handles: +/// - Event accumulation +/// - Statistics calculation +/// - Thread-safe state management +/// +/// Subclasses should override `formatEvent` to provide custom output formatting. +open class BaseProgressConsumer: ProgressConsumer, @unchecked Sendable { + public let configuration: Configuration + private let lock = NSLock() + + // Statistics tracking + private var accumulatedEvents: [BuildEvent] = [] + private var buildStartTime: Date? + private var buildEndTime: Date? + private var buildSuccess: Bool? + private var totalOperationCount = 0 + private var executedOperationCount = 0 + private var cacheHitCount = 0 + private var failedOperationCount = 0 + private var stageStats: [String: MutableStageStats] = [:] + + private struct MutableStageStats { + var name: String + var startTime: Date? + var endTime: Date? + var operationCount = 0 + var cacheHits = 0 + var failures = 0 + } + + public required init(configuration: Configuration) { + self.configuration = configuration + } + + public func consume(reporter: Reporter) async throws { + for await event in reporter.stream { + try await handle(event) + } + } + + public func handle(_ event: BuildEvent) async throws { + // Store event and update statistics + lock.withLock { + accumulatedEvents.append(event) + updateStatistics(event) + } + + // Let subclass format the output + try await formatAndOutput(event) + } + + /// Subclasses must implement this to format and output events. + open func formatAndOutput(_ event: BuildEvent) async throws { + fatalError("Subclasses must implement formatAndOutput(_:)") + } + + public func getStatistics() -> BuildStatistics { + lock.withLock { + let stageStatistics = stageStats.mapValues { stats in + StageStatistics( + name: stats.name, + startTime: stats.startTime, + endTime: stats.endTime, + operationCount: stats.operationCount, + cacheHits: stats.cacheHits, + failures: stats.failures + ) + } + + return BuildStatistics( + startTime: buildStartTime, + endTime: buildEndTime, + success: buildSuccess, + totalOperations: totalOperationCount, + executedOperations: executedOperationCount, + cacheHits: cacheHitCount, + failedOperations: failedOperationCount, + totalStages: stageStats.count, + stageStatistics: stageStatistics, + events: accumulatedEvents + ) + } + } + + public func getEvents() -> [BuildEvent] { + lock.withLock { + accumulatedEvents + } + } + + private func updateStatistics(_ event: BuildEvent) { + switch event { + case .buildStarted(let totalOps, _, let timestamp): + buildStartTime = timestamp + totalOperationCount = totalOps + + case .buildCompleted(let success, let timestamp): + buildEndTime = timestamp + buildSuccess = success + + case .stageStarted(let stageName, let timestamp): + if stageStats[stageName] == nil { + stageStats[stageName] = MutableStageStats(name: stageName) + } + stageStats[stageName]?.startTime = timestamp + + case .stageCompleted(let stageName, let timestamp): + if stageStats[stageName] == nil { + stageStats[stageName] = MutableStageStats(name: stageName) + } + stageStats[stageName]?.endTime = timestamp + + case .operationStarted(let context): + if let stage = context.stageId { + if stageStats[stage] == nil { + stageStats[stage] = MutableStageStats(name: stage) + } + stageStats[stage]?.operationCount += 1 + } + + case .operationFinished: + executedOperationCount += 1 + + case .operationFailed(let context, _): + failedOperationCount += 1 + if let stage = context.stageId { + if stageStats[stage] == nil { + stageStats[stage] = MutableStageStats(name: stage) + } + stageStats[stage]?.failures += 1 + } + + case .operationCacheHit(let context): + cacheHitCount += 1 + if let stage = context.stageId { + if stageStats[stage] == nil { + stageStats[stage] = MutableStageStats(name: stage) + } + stageStats[stage]?.cacheHits += 1 + } + + case .operationProgress, .operationLog: + break // These don't affect statistics + + case .irEvent(_, _): + // Track IR events if needed in the future + break + } + } +} + +// Helper extension for thread-safe lock usage +extension NSLock { + fileprivate func withLock(_ body: () throws -> T) rethrows -> T { + lock() + defer { unlock() } + return try body() + } +} diff --git a/Sources/NativeBuilder/ContainerBuildReporting/BuildEvent.swift b/Sources/NativeBuilder/ContainerBuildReporting/BuildEvent.swift new file mode 100644 index 00000000..75b53904 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildReporting/BuildEvent.swift @@ -0,0 +1,189 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// Represents all types of events that can occur during a build. +/// +/// Design rationale: +/// - Single enum to capture all build activity (operations, logs, progress) +/// - Each case carries relevant context and data +/// - Sendable and Codable for thread safety and serialization +/// - Extensible for future event types +public enum BuildEvent: Sendable, Codable { + // MARK: - Build Lifecycle Events + + /// Build has started + case buildStarted(totalOperations: Int, stages: Int, timestamp: Date) + + /// Build has completed + case buildCompleted(success: Bool, timestamp: Date) + + // MARK: - Stage Events + + /// A build stage has started + case stageStarted(stageName: String, timestamp: Date) + + /// A build stage has completed + case stageCompleted(stageName: String, timestamp: Date) + + // MARK: - Operation Events + + /// An operation has started executing + case operationStarted(context: ReportContext) + + /// An operation has finished successfully + case operationFinished(context: ReportContext, duration: TimeInterval) + + /// An operation has failed + case operationFailed(context: ReportContext, error: BuildEventError) + + /// An operation was satisfied from cache + case operationCacheHit(context: ReportContext) + + /// Progress update for a long-running operation + case operationProgress(context: ReportContext, fraction: Double) + + /// Log output from an operation + case operationLog(context: ReportContext, message: String) + + // MARK: - IR Events + + /// IR construction or analysis event + case irEvent(context: ReportContext, type: IREventType) +} + +/// Context information for events. +/// +/// Design rationale: +/// - Provides provenance for each event +/// - Enables grouping and correlation of events +/// - Rich context for UI/logging decisions +/// - Source mapping for precise error location +public struct ReportContext: Sendable, Codable { + /// Unique identifier for the node (if applicable) + public let nodeId: UUID? + + /// Identifier for the build stage (if applicable) + public let stageId: String? + + /// Human-readable description + public let description: String + + /// Timestamp when the event was generated + public let timestamp: Date + + /// Source location mapping (if available) + public let sourceMap: SourceMap? + + public init( + nodeId: UUID? = nil, + stageId: String? = nil, + description: String, + timestamp: Date = Date(), + sourceMap: SourceMap? = nil + ) { + self.nodeId = nodeId + self.stageId = stageId + self.description = description + self.timestamp = timestamp + self.sourceMap = sourceMap + } + + /// Convenience init for operation events (backwards compatibility) + public init( + nodeId: UUID, + stageId: String, + operationDescription: String, + timestamp: Date = Date() + ) { + self.init( + nodeId: nodeId, + stageId: stageId, + description: operationDescription, + timestamp: timestamp + ) + } +} + +/// Source location information for precise error reporting +public struct SourceMap: Sendable, Codable { + /// Source file path (e.g., Dockerfile path) + public let file: String? + + /// Line number (1-based) + public let line: Int? + + /// Column number (1-based) + public let column: Int? + + /// Source text snippet for context + public let snippet: String? + + public init(file: String? = nil, line: Int? = nil, column: Int? = nil, snippet: String? = nil) { + self.file = file + self.line = line + self.column = column + self.snippet = snippet + } +} + +/// Types of IR events +public enum IREventType: String, Sendable, Codable { + case graphStarted = "graph_started" + case graphCompleted = "graph_completed" + case stageAdded = "stage_added" + case nodeAdded = "node_added" + case analyzing = "analyzing" + case validating = "validating" + case error = "error" + case warning = "warning" +} + +/// Error information for build events. +/// +/// Design rationale: +/// - Structured error representation for serialization +/// - Captures error type and description +/// - Extensible with diagnostics +public struct BuildEventError: Sendable, Codable { + /// The type of failure + public let type: FailureType + + /// Human-readable error description + public let description: String + + /// Additional diagnostic information + public let diagnostics: [String: String]? + + public init( + type: FailureType, + description: String, + diagnostics: [String: String]? = nil + ) { + self.type = type + self.description = description + self.diagnostics = diagnostics + } + + public enum FailureType: String, Sendable, Codable { + case executionFailed + case cancelled + case invalidConfiguration + case timeout + case resourceExhausted + } +} diff --git a/Sources/NativeBuilder/ContainerBuildReporting/BuildStatistics.swift b/Sources/NativeBuilder/ContainerBuildReporting/BuildStatistics.swift new file mode 100644 index 00000000..407bb6b8 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildReporting/BuildStatistics.swift @@ -0,0 +1,123 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// Statistics collected during a build execution. +public struct BuildStatistics: Sendable { + /// When the build started + public let startTime: Date? + + /// When the build completed + public let endTime: Date? + + /// Total build duration + public var duration: TimeInterval? { + guard let start = startTime, let end = endTime else { return nil } + return end.timeIntervalSince(start) + } + + /// Whether the build succeeded + public let success: Bool? + + /// Total number of operations + public let totalOperations: Int + + /// Number of operations executed + public let executedOperations: Int + + /// Number of cache hits + public let cacheHits: Int + + /// Number of failed operations + public let failedOperations: Int + + /// Number of stages + public let totalStages: Int + + /// Per-stage statistics + public let stageStatistics: [String: StageStatistics] + + /// All events that occurred during the build + public let events: [BuildEvent] + + public init( + startTime: Date? = nil, + endTime: Date? = nil, + success: Bool? = nil, + totalOperations: Int = 0, + executedOperations: Int = 0, + cacheHits: Int = 0, + failedOperations: Int = 0, + totalStages: Int = 0, + stageStatistics: [String: StageStatistics] = [:], + events: [BuildEvent] = [] + ) { + self.startTime = startTime + self.endTime = endTime + self.success = success + self.totalOperations = totalOperations + self.executedOperations = executedOperations + self.cacheHits = cacheHits + self.failedOperations = failedOperations + self.totalStages = totalStages + self.stageStatistics = stageStatistics + self.events = events + } +} + +/// Statistics for a single build stage. +public struct StageStatistics: Sendable { + /// Stage name + public let name: String + + /// When the stage started + public let startTime: Date? + + /// When the stage completed + public let endTime: Date? + + /// Stage duration + public var duration: TimeInterval? { + guard let start = startTime, let end = endTime else { return nil } + return end.timeIntervalSince(start) + } + + /// Number of operations in this stage + public let operationCount: Int + + /// Number of cache hits in this stage + public let cacheHits: Int + + /// Number of failures in this stage + public let failures: Int + + public init( + name: String, + startTime: Date? = nil, + endTime: Date? = nil, + operationCount: Int = 0, + cacheHits: Int = 0, + failures: Int = 0 + ) { + self.name = name + self.startTime = startTime + self.endTime = endTime + self.operationCount = operationCount + self.cacheHits = cacheHits + self.failures = failures + } +} diff --git a/Sources/NativeBuilder/ContainerBuildReporting/JSONProgressConsumer.swift b/Sources/NativeBuilder/ContainerBuildReporting/JSONProgressConsumer.swift new file mode 100644 index 00000000..cc7ec31e --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildReporting/JSONProgressConsumer.swift @@ -0,0 +1,312 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// JSON progress consumer that outputs newline-delimited JSON events. +/// +/// This consumer is useful for: +/// - Machine-readable output +/// - Integration with external tools +/// - Structured logging systems +/// +/// Example output: +/// ``` +/// {"type":"operation_started","operation":"#1","description":"[internal] load metadata for alpine:latest","timestamp":"2025-01-09T10:30:45.123Z"} +/// {"type":"operation_finished","operation":"#1","duration":0.5,"timestamp":"2025-01-09T10:30:45.623Z"} +/// ``` +public final class JSONProgressConsumer: BaseProgressConsumer, @unchecked Sendable { + public struct Configuration: Sendable { + /// File handle to write output to (default: stdout) + public let output: FileHandle + + /// Pretty print JSON (with indentation) + public let prettyPrint: Bool + + public init( + output: FileHandle = .standardOutput, + prettyPrint: Bool = false + ) { + self.output = output + self.prettyPrint = prettyPrint + } + } + + private let encoder: JSONEncoder + private let lock = NSLock() + private var operationNumbers: [UUID: Int] = [:] + private var nextOperationNumber = 1 + + public required init(configuration: Configuration) { + self.encoder = JSONEncoder() + if configuration.prettyPrint { + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + } + encoder.dateEncodingStrategy = .iso8601 + super.init(configuration: configuration) + } + + override public func formatAndOutput(_ event: BuildEvent) async throws { + try lock.withLock { + if let output = try formatEventAsJSON(event) { + try configuration.output.write(contentsOf: output) + try configuration.output.write(contentsOf: "\n".data(using: .utf8)!) + } + } + } + + private func formatEventAsJSON(_ event: BuildEvent) throws -> Data? { + let jsonEvent: JSONEvent? = { + switch event { + case .buildStarted(let totalOps, let stages, let timestamp): + return JSONEvent( + type: "build_started", + timestamp: timestamp, + data: [ + "total_operations": totalOps, + "stages": stages, + ] + ) + + case .buildCompleted(let success, let timestamp): + return JSONEvent( + type: "build_completed", + timestamp: timestamp, + data: ["success": success] + ) + + case .stageStarted(let stageName, let timestamp): + return JSONEvent( + type: "stage_started", + timestamp: timestamp, + data: ["stage": stageName] + ) + + case .stageCompleted(let stageName, let timestamp): + return JSONEvent( + type: "stage_completed", + timestamp: timestamp, + data: ["stage": stageName] + ) + + case .operationStarted(let context): + guard let nodeId = context.nodeId else { return nil } + let number = assignOperationNumber(for: nodeId) + var data: [String: Any] = [ + "operation": "#\(number)", + "description": context.description, + ] + if let stageId = context.stageId { + data["stage"] = stageId + } + return JSONEvent( + type: "operation_started", + timestamp: context.timestamp, + data: data + ) + + case .operationFinished(let context, let duration): + guard let nodeId = context.nodeId, + let number = operationNumbers[nodeId] + else { return nil } + return JSONEvent( + type: "operation_finished", + timestamp: context.timestamp, + data: [ + "operation": "#\(number)", + "duration": duration, + ] + ) + + case .operationFailed(let context, let error): + guard let nodeId = context.nodeId, + let number = operationNumbers[nodeId] + else { return nil } + return JSONEvent( + type: "operation_failed", + timestamp: context.timestamp, + data: [ + "operation": "#\(number)", + "error": error.description, + "error_type": error.type.rawValue, + ] + ) + + case .operationCacheHit(let context): + guard let nodeId = context.nodeId else { return nil } + let number = assignOperationNumber(for: nodeId) + return JSONEvent( + type: "operation_cache_hit", + timestamp: context.timestamp, + data: [ + "operation": "#\(number)", + "description": context.description, + ] + ) + + case .operationProgress(let context, let fraction): + guard let nodeId = context.nodeId, + let number = operationNumbers[nodeId] + else { return nil } + return JSONEvent( + type: "operation_progress", + timestamp: context.timestamp, + data: [ + "operation": "#\(number)", + "progress": fraction, + ] + ) + + case .operationLog(let context, let message): + guard let nodeId = context.nodeId, + let number = operationNumbers[nodeId] + else { return nil } + return JSONEvent( + type: "operation_log", + timestamp: context.timestamp, + data: [ + "operation": "#\(number)", + "message": message, + ] + ) + + case .irEvent(let context, let type): + var data: [String: Any] = [ + "event_type": type.rawValue, + "description": context.description, + ] + + if let nodeId = context.nodeId { + data["node_id"] = nodeId.uuidString + } + if let stageId = context.stageId { + data["stage_id"] = stageId + } + if let sourceMap = context.sourceMap { + var mapData: [String: Any] = [:] + if let file = sourceMap.file { mapData["file"] = file } + if let line = sourceMap.line { mapData["line"] = line } + if let column = sourceMap.column { mapData["column"] = column } + if let snippet = sourceMap.snippet { mapData["snippet"] = snippet } + data["source_map"] = mapData + } + + return JSONEvent( + type: "ir_event", + timestamp: context.timestamp, + data: data + ) + } + }() + + if let jsonEvent = jsonEvent { + return try encoder.encode(jsonEvent) + } + return nil + } + + private func assignOperationNumber(for nodeId: UUID) -> Int { + if let existing = operationNumbers[nodeId] { + return existing + } + let number = nextOperationNumber + operationNumbers[nodeId] = number + nextOperationNumber += 1 + return number + } + + private struct JSONEvent: Encodable { + let type: String + let timestamp: Date + let data: [String: Any] + + enum CodingKeys: String, CodingKey { + case type + case timestamp + } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(type, forKey: .type) + try container.encode(timestamp, forKey: .timestamp) + + // Encode the dynamic data fields + var dataContainer = encoder.container(keyedBy: DynamicCodingKey.self) + for (key, value) in data { + let codingKey = DynamicCodingKey(stringValue: key) + switch value { + case let intValue as Int: + try dataContainer.encode(intValue, forKey: codingKey) + case let doubleValue as Double: + try dataContainer.encode(doubleValue, forKey: codingKey) + case let stringValue as String: + try dataContainer.encode(stringValue, forKey: codingKey) + case let boolValue as Bool: + try dataContainer.encode(boolValue, forKey: codingKey) + case let dictValue as [String: Any]: + // For dictionary values, we need to encode them properly + let nestedContainer = dataContainer.nestedContainer(keyedBy: DynamicCodingKey.self, forKey: codingKey) + try encodeNestedDictionary(dictValue, to: nestedContainer) + default: + // Skip unsupported types + break + } + } + } + + private func encodeNestedDictionary(_ dict: [String: Any], to container: KeyedEncodingContainer) throws { + var container = container + for (key, value) in dict { + let codingKey = DynamicCodingKey(stringValue: key) + switch value { + case let intValue as Int: + try container.encode(intValue, forKey: codingKey) + case let doubleValue as Double: + try container.encode(doubleValue, forKey: codingKey) + case let stringValue as String: + try container.encode(stringValue, forKey: codingKey) + case let boolValue as Bool: + try container.encode(boolValue, forKey: codingKey) + default: + // Skip unsupported types + break + } + } + } + } + + private struct DynamicCodingKey: CodingKey { + let stringValue: String + let intValue: Int? = nil + + init(stringValue: String) { + self.stringValue = stringValue + } + + init?(intValue: Int) { + return nil + } + } +} + +// Helper extension for thread-safe lock usage +extension NSLock { + fileprivate func withLock(_ body: () throws -> T) rethrows -> T { + lock() + defer { unlock() } + return try body() + } +} diff --git a/Sources/NativeBuilder/ContainerBuildReporting/PlainProgressConsumer.swift b/Sources/NativeBuilder/ContainerBuildReporting/PlainProgressConsumer.swift new file mode 100644 index 00000000..29bef942 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildReporting/PlainProgressConsumer.swift @@ -0,0 +1,225 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// Plain text progress consumer that mimics BuildKit's progress=plain format. +/// +/// Output format matches BuildKit's clean, numbered operation display: +/// ``` +/// #1 [internal] load build definition +/// #1 DONE 0.1s +/// +/// #2 [base 1/3] FROM alpine:latest +/// #2 CACHED +/// +/// #3 [base 2/3] RUN apk add --no-cache git +/// #3 0.245 fetch https://dl-cdn.alpinelinux.org/alpine/... +/// #3 DONE 1.2s +/// ``` +public final class PlainProgressConsumer: BaseProgressConsumer, @unchecked Sendable { + public struct Configuration: Sendable { + /// File handle to write output to (default: stdout) + public let output: FileHandle + + public init( + output: FileHandle = .standardOutput, + includeEventIds: Bool = false, + timestampFormat: TimestampFormat = .iso8601 + ) { + self.output = output + } + + public enum TimestampFormat: Sendable { + case iso8601 + case unix + case relative(startTime: Date) + } + } + + private let lock = NSLock() + + // State tracking for BuildKit-style formatting + private var operationNumbers: [UUID: Int] = [:] + private var operationStartTimes: [UUID: Date] = [:] + private var nextOperationNumber = 1 + + public required init(configuration: Configuration) { + super.init(configuration: configuration) + } + + override public func formatAndOutput(_ event: BuildEvent) async throws { + let output = formatEvent(event) + if !output.isEmpty { + let data = (output + "\n").data(using: .utf8) ?? Data() + try configuration.output.write(contentsOf: data) + } + } + + private func formatEvent(_ event: BuildEvent) -> String { + lock.withLock { + switch event { + case .buildStarted: + return "" // BuildKit doesn't show explicit build start + + case .buildCompleted: + return "" // Let caller handle build completion messages + + case .stageStarted: + return "" // Stages are implicit in operation descriptions + + case .stageCompleted: + return "" // Stages are implicit in operation descriptions + + case .operationStarted(let context): + guard let nodeId = context.nodeId else { return "" } + let number = assignOperationNumber(for: nodeId) + operationStartTimes[nodeId] = context.timestamp + return "#\(number) \(formatOperationDescription(context.description, stage: context.stageId))" + + case .operationFinished(let context, _): + guard let nodeId = context.nodeId, + let number = operationNumbers[nodeId], + let startTime = operationStartTimes[nodeId] + else { + return "" + } + let duration = context.timestamp.timeIntervalSince(startTime) + operationStartTimes.removeValue(forKey: nodeId) + return "#\(number) DONE \(formatDuration(duration))" + + case .operationFailed(let context, let error): + guard let nodeId = context.nodeId, + let number = operationNumbers[nodeId] + else { return "" } + operationStartTimes.removeValue(forKey: nodeId) + return "#\(number) ERROR: \(error.description)" + + case .operationCacheHit(let context): + guard let nodeId = context.nodeId else { return "" } + + // Check if this operation was already started + let wasStarted = operationNumbers[nodeId] != nil + let number = assignOperationNumber(for: nodeId) + operationStartTimes.removeValue(forKey: nodeId) + + guard wasStarted else { + // Cache hit without prior start - show both description and CACHED + let description = formatOperationDescription(context.description, stage: context.stageId) + return "#\(number) \(description)\n#\(number) CACHED" + } + // Just show CACHED - the operation description was already shown + return "#\(number) CACHED" + + case .operationProgress(let context, let fraction): + guard let nodeId = context.nodeId, + let number = operationNumbers[nodeId] + else { return "" } + let percentage = Int(fraction * 100) + return "#\(number) \(percentage)% complete" + + case .operationLog(let context, let message): + guard let nodeId = context.nodeId, + let number = operationNumbers[nodeId], + let startTime = operationStartTimes[nodeId] + else { + return "" + } + let elapsed = context.timestamp.timeIntervalSince(startTime) + // Format log lines like BuildKit: #N elapsed message + return "#\(number) \(String(format: "%.3f", elapsed)) \(message)" + + case .irEvent(let context, let type): + // Format IR events based on type + switch type { + case .graphStarted, .graphCompleted: + return "" // Don't show graph-level events in plain output + case .stageAdded: + return "" // Stage creation is implicit in BuildKit output + case .nodeAdded: + return "" // Node addition is shown when executed + case .analyzing: + return "=> \(context.description)" + case .validating: + return "=> \(context.description)" + case .error: + if let sourceMap = context.sourceMap { + return "ERROR: \(context.description) at \(sourceMap.file ?? "unknown"):\(sourceMap.line ?? 0)" + } + return "ERROR: \(context.description)" + case .warning: + if let sourceMap = context.sourceMap { + return "WARNING: \(context.description) at \(sourceMap.file ?? "unknown"):\(sourceMap.line ?? 0)" + } + return "WARNING: \(context.description)" + } + } + } + } + + private func assignOperationNumber(for nodeId: UUID) -> Int { + if let existing = operationNumbers[nodeId] { + return existing + } + let number = nextOperationNumber + operationNumbers[nodeId] = number + nextOperationNumber += 1 + return number + } + + private func formatOperationDescription(_ description: String, stage: String? = nil) -> String { + // Transform operation descriptions to BuildKit style + // Examples: + // "FROM alpine:latest" -> "[internal] load metadata for alpine:latest" + // "RUN apk add git" -> "[stage-name] RUN apk add git" + + if description.hasPrefix("FROM ") { + let imageName = description.replacingOccurrences(of: "FROM ", with: "") + return "[internal] load metadata for \(imageName)" + } else if description.hasPrefix("BaseImage:") { + // Transform our internal representation + let imageName = description.replacingOccurrences(of: "BaseImage: ", with: "") + return "[internal] load metadata for \(imageName)" + } else if let stage = stage { + // Clean up stage name (remove "stage-" prefix if it's a UUID) + let stageName = stage.hasPrefix("stage-") && stage.count > 12 ? "stage" : stage + return "[\(stageName)] \(description)" + } else { + return "[stage] \(description)" + } + } + + private func formatDuration(_ duration: TimeInterval) -> String { + if duration < 1.0 { + return String(format: "%.1fs", duration) + } else if duration < 60.0 { + return String(format: "%.1fs", duration) + } else { + let minutes = Int(duration / 60) + let seconds = Int(duration.truncatingRemainder(dividingBy: 60)) + return String(format: "%dm%ds", minutes, seconds) + } + } +} + +// Helper extension for thread-safe lock usage +extension NSLock { + fileprivate func withLock(_ body: () throws -> T) rethrows -> T { + lock() + defer { unlock() } + return try body() + } +} diff --git a/Sources/NativeBuilder/ContainerBuildReporting/ProgressConsumer.swift b/Sources/NativeBuilder/ContainerBuildReporting/ProgressConsumer.swift new file mode 100644 index 00000000..910f12a6 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildReporting/ProgressConsumer.swift @@ -0,0 +1,44 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// Protocol for consuming and displaying build progress events. +/// +/// Design rationale: +/// - Protocol-based design allows for different output formats +/// - Async/await support for streaming events +/// - Flexible configuration for different environments +/// - Built-in statistics tracking for all consumers +public protocol ProgressConsumer: Sendable { + /// Configuration for the progress consumer + associatedtype Configuration: Sendable + + /// Initialize with configuration + init(configuration: Configuration) + + /// Consume events from the reporter and display progress + func consume(reporter: Reporter) async throws + + /// Handle a single event (for testing or custom implementations) + func handle(_ event: BuildEvent) async throws + + /// Get the accumulated build statistics + func getStatistics() -> BuildStatistics + + /// Get all accumulated events + func getEvents() -> [BuildEvent] +} diff --git a/Sources/NativeBuilder/ContainerBuildReporting/ReportableError.swift b/Sources/NativeBuilder/ContainerBuildReporting/ReportableError.swift new file mode 100644 index 00000000..1872aeee --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildReporting/ReportableError.swift @@ -0,0 +1,258 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// Protocol for errors that can be reported through the build event system. +/// +/// Design rationale: +/// - Enforces consistent error formatting across the system +/// - Provides structured error information for reporting +/// - Enables centralized error formatting control +public protocol ReportableError: Error { + /// The category of the error + var errorCategory: ErrorCategory { get } + + /// A short, user-friendly description of what went wrong + var shortDescription: String { get } + + /// Detailed information about the error for debugging + var detailedDescription: String? { get } + + /// Structured diagnostic information + var diagnostics: ErrorDiagnostics { get } + + /// The underlying error that caused this error (if any) + var underlyingError: Error? { get } + + /// Convert to BuildEventError for reporting + func toBuildEventError() -> BuildEventError +} + +/// Categories of errors that can occur during build +public enum ErrorCategory: String, Sendable, Codable { + // Execution errors + case executionFailed = "execution_failed" + case commandNotFound = "command_not_found" + case permissionDenied = "permission_denied" + case timeout = "timeout" + case cancelled = "cancelled" + + // Configuration errors + case invalidConfiguration = "invalid_configuration" + case missingDependency = "missing_dependency" + case incompatiblePlatform = "incompatible_platform" + + // Resource errors + case resourceExhausted = "resource_exhausted" + case diskFull = "disk_full" + case memoryExhausted = "memory_exhausted" + + // I/O errors + case fileNotFound = "file_not_found" + case fileAccessDenied = "file_access_denied" + case networkError = "network_error" + + // Parse/validation errors + case syntaxError = "syntax_error" + case validationError = "validation_error" + case unsupportedFeature = "unsupported_feature" + + // Cache errors + case cacheCorrupted = "cache_corrupted" + case cacheMiss = "cache_miss" + + // Unknown + case unknown = "unknown" +} + +/// Structured diagnostic information for errors +public struct ErrorDiagnostics: Sendable, Codable { + /// The operation that was being performed + public let operation: String? + + /// The file or resource involved + public let path: String? + + /// Line number (for parse errors) + public let line: Int? + + /// Column number (for parse errors) + public let column: Int? + + /// Exit code (for execution errors) + public let exitCode: Int? + + /// Working directory + public let workingDirectory: String? + + /// Relevant environment variables + public let environment: [String: String]? + + /// Recent log output + public let recentLogs: [String]? + + /// Additional context-specific information + public let additionalInfo: [String: String]? + + public init( + operation: String? = nil, + path: String? = nil, + line: Int? = nil, + column: Int? = nil, + exitCode: Int? = nil, + workingDirectory: String? = nil, + environment: [String: String]? = nil, + recentLogs: [String]? = nil, + additionalInfo: [String: String]? = nil + ) { + self.operation = operation + self.path = path + self.line = line + self.column = column + self.exitCode = exitCode + self.workingDirectory = workingDirectory + self.environment = environment + self.recentLogs = recentLogs + self.additionalInfo = additionalInfo + } + + /// Convert to flat dictionary for BuildEventError + func toDictionary() -> [String: String] { + var dict: [String: String] = [:] + + if let operation = operation { + dict["operation"] = operation + } + if let path = path { + dict["path"] = path + } + if let line = line { + dict["line"] = String(line) + } + if let column = column { + dict["column"] = String(column) + } + if let exitCode = exitCode { + dict["exitCode"] = String(exitCode) + } + if let workingDirectory = workingDirectory { + dict["workingDirectory"] = workingDirectory + } + if let environment = environment { + for (key, value) in environment { + dict["env.\(key)"] = value + } + } + if let recentLogs = recentLogs, !recentLogs.isEmpty { + dict["recentLogs"] = recentLogs.joined(separator: "\n") + } + if let additionalInfo = additionalInfo { + for (key, value) in additionalInfo { + dict[key] = value + } + } + + return dict + } +} + +// MARK: - Default Implementation + +extension ReportableError { + /// Default implementation that converts to BuildEventError + public func toBuildEventError() -> BuildEventError { + // Map error category to BuildEventError.FailureType + let failureType: BuildEventError.FailureType + switch errorCategory { + case .executionFailed, .commandNotFound, .permissionDenied: + failureType = .executionFailed + case .timeout: + failureType = .timeout + case .cancelled: + failureType = .cancelled + case .invalidConfiguration, .missingDependency, .incompatiblePlatform, + .syntaxError, .validationError, .unsupportedFeature: + failureType = .invalidConfiguration + case .resourceExhausted, .diskFull, .memoryExhausted: + failureType = .resourceExhausted + case .fileNotFound, .fileAccessDenied, .networkError, + .cacheCorrupted, .cacheMiss, .unknown: + failureType = .executionFailed + } + + // Build description + var description = shortDescription + if let detailed = detailedDescription { + description += ". \(detailed)" + } + if let underlying = underlyingError { + description += ". Caused by: \(underlying.localizedDescription)" + } + + return BuildEventError( + type: failureType, + description: description, + diagnostics: diagnostics.toDictionary() + ) + } + + /// Default values for optional properties + public var detailedDescription: String? { nil } + public var underlyingError: Error? { nil } +} + +// MARK: - Generic Error Extension + +/// Extension to make any Error reportable with basic information +extension Error { + /// Convert any error to a ReportableError + public func asReportableError() -> ReportableError { + if let reportable = self as? ReportableError { + return reportable + } + return GenericReportableError(underlying: self) + } +} + +/// Wrapper for non-ReportableError errors +private struct GenericReportableError: ReportableError { + let underlying: Error + + var errorCategory: ErrorCategory { .unknown } + + var shortDescription: String { + (underlying as NSError).localizedDescription + } + + var diagnostics: ErrorDiagnostics { + var additionalInfo: [String: String] = [:] + + let nsError = underlying as NSError + additionalInfo["domain"] = nsError.domain + additionalInfo["code"] = String(nsError.code) + + for (key, value) in nsError.userInfo { + if let stringValue = value as? String { + additionalInfo["userInfo.\(key)"] = stringValue + } + } + + return ErrorDiagnostics(additionalInfo: additionalInfo) + } + + var underlyingError: Error? { underlying } +} diff --git a/Sources/NativeBuilder/ContainerBuildReporting/Reporter.swift b/Sources/NativeBuilder/ContainerBuildReporting/Reporter.swift new file mode 100644 index 00000000..e4dec8fc --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildReporting/Reporter.swift @@ -0,0 +1,46 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// Central event hub for build progress reporting. +/// +/// Design rationale: +/// - Actor-based for thread safety without manual locking +/// - AsyncStream for real-time event consumption +/// - Bounded buffer to prevent unbounded memory growth +/// - Single source of truth for all build events +public actor Reporter { + private let continuation: AsyncStream.Continuation + public nonisolated let stream: AsyncStream + + /// Initialize with a buffer size for the event stream + public init(bufferSize: Int = 100) { + (self.stream, self.continuation) = AsyncStream.makeStream( + bufferingPolicy: .bufferingNewest(bufferSize) + ) + } + + /// Report a new event + public func report(_ event: BuildEvent) { + continuation.yield(event) + } + + /// Finish the event stream (no more events will be reported) + public func finish() { + continuation.finish() + } +} diff --git a/Sources/NativeBuilder/ContainerBuildSnapshotter/Snapshotter.swift b/Sources/NativeBuilder/ContainerBuildSnapshotter/Snapshotter.swift new file mode 100644 index 00000000..2a842fa7 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildSnapshotter/Snapshotter.swift @@ -0,0 +1,194 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import Foundation + +/// Manages filesystem snapshots during build execution. +/// +/// The snapshotter is responsible for creating and managing filesystem +/// snapshots that represent the state at different points in the build. +public protocol Snapshotter: Sendable { + /// Create a new snapshot from the current state. + /// + /// - Parameters: + /// - parent: The parent snapshot to base this on + /// - changes: The filesystem changes to apply + /// - Returns: The new snapshot + func createSnapshot( + from parent: Snapshot?, + applying changes: FilesystemChanges + ) async throws -> Snapshot + + /// Prepare a snapshot for use (e.g., mount it). + /// + /// - Parameter snapshot: The snapshot to prepare + /// - Returns: A handle to the prepared snapshot + func prepare(_ snapshot: Snapshot) async throws -> SnapshotHandle + + /// Commit a snapshot, making it permanent. + /// + /// - Parameter snapshot: The snapshot to commit + /// - Returns: The committed snapshot with final digest + func commit(_ snapshot: Snapshot) async throws -> Snapshot + + /// Remove a snapshot. + /// + /// - Parameter snapshot: The snapshot to remove + func remove(_ snapshot: Snapshot) async throws + + /// Get the diff between two snapshots. + /// + /// - Parameters: + /// - from: The base snapshot + /// - to: The target snapshot + /// - Returns: The filesystem changes between snapshots + func diff(from: Snapshot?, to: Snapshot) async throws -> FilesystemChanges +} + +/// A handle to a prepared snapshot. +public struct SnapshotHandle: Sendable { + /// The snapshot being handled. + public let snapshot: Snapshot + + /// The mount point or working directory for the snapshot. + public let path: String + + /// Cleanup function to call when done. + private let cleanup: @Sendable () async -> Void + + public init( + snapshot: Snapshot, + path: String, + cleanup: @escaping @Sendable () async -> Void + ) { + self.snapshot = snapshot + self.path = path + self.cleanup = cleanup + } + + /// Clean up the prepared snapshot. + public func close() async { + await cleanup() + } +} + +/// A memory-based snapshotter for development/testing. +public actor MemorySnapshotter: Snapshotter { + private var snapshots: [UUID: SnapshotData] = [:] + private var nextId = 0 + + private struct SnapshotData { + let snapshot: Snapshot + let changes: FilesystemChanges + var committed: Bool + } + + public init() {} + + public func createSnapshot( + from parent: Snapshot?, + applying changes: FilesystemChanges + ) async throws -> Snapshot { + nextId += 1 + let id = UUID() + + // Create a fake 32-byte digest for sha256 + var digestBytes = Data(count: 32) + digestBytes.withUnsafeMutableBytes { bytes in + if let baseAddress = bytes.baseAddress { + memset(baseAddress, Int32(nextId % 256), 32) + } + } + let digest = try Digest(algorithm: .sha256, bytes: digestBytes) + + let snapshot = Snapshot( + id: id, + digest: digest, + size: abs(changes.sizeChange), + parent: parent?.id + ) + + snapshots[id] = SnapshotData( + snapshot: snapshot, + changes: changes, + committed: false + ) + + return snapshot + } + + public func prepare(_ snapshot: Snapshot) async throws -> SnapshotHandle { + guard snapshots[snapshot.id] != nil else { + throw SnapshotError.notFound(snapshot.id) + } + + // For memory snapshotter, we just return a temp directory + let path = "/tmp/snapshot-\(snapshot.id)" + + return SnapshotHandle( + snapshot: snapshot, + path: path, + cleanup: { [weak self] in + // In a real implementation, this would unmount/cleanup + _ = self + } + ) + } + + public func commit(_ snapshot: Snapshot) async throws -> Snapshot { + guard var data = snapshots[snapshot.id] else { + throw SnapshotError.notFound(snapshot.id) + } + + // Mark as committed + data.committed = true + snapshots[snapshot.id] = data + + return snapshot + } + + public func remove(_ snapshot: Snapshot) async throws { + snapshots.removeValue(forKey: snapshot.id) + } + + public func diff(from base: Snapshot?, to target: Snapshot) async throws -> FilesystemChanges { + guard let targetData = snapshots[target.id] else { + throw SnapshotError.notFound(target.id) + } + + // For simplicity, just return the target's changes + return targetData.changes + } +} + +/// Errors that can occur during snapshot operations. +public enum SnapshotError: LocalizedError { + case notFound(UUID) + case invalidParent(UUID) + case commitFailed(Error) + + public var errorDescription: String? { + switch self { + case .notFound(let id): + return "Snapshot not found: \(id)" + case .invalidParent(let id): + return "Invalid parent snapshot: \(id)" + case .commitFailed(let error): + return "Failed to commit snapshot: \(error.localizedDescription)" + } + } +} diff --git a/Sources/NativeBuilder/ContainerBuildSnapshotter/Types.swift b/Sources/NativeBuilder/ContainerBuildSnapshotter/Types.swift new file mode 100644 index 00000000..e6b01835 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildSnapshotter/Types.swift @@ -0,0 +1,88 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import Foundation + +/// A filesystem snapshot representing state at a point in the build. +public struct Snapshot: Sendable, Codable { + /// Unique identifier for this snapshot. + public let id: UUID + + /// The digest of the snapshot content. + public let digest: Digest + + /// Size of the snapshot in bytes. + public let size: Int64 + + /// Parent snapshot (if any). + public let parent: UUID? + + /// Timestamp when the snapshot was created. + public let createdAt: Date + + public init( + id: UUID = UUID(), + digest: Digest, + size: Int64, + parent: UUID? = nil, + createdAt: Date = Date() + ) { + self.id = id + self.digest = digest + self.size = size + self.parent = parent + self.createdAt = createdAt + } +} + +/// Describes filesystem changes made by an operation. +public struct FilesystemChanges: Sendable, Codable { + /// Files that were added. + public let added: Set + + /// Files that were modified. + public let modified: Set + + /// Files that were deleted. + public let deleted: Set + + /// Files that were removed (alias for deleted). + public var removed: Set { deleted } + + /// Total size change in bytes. + public let sizeChange: Int64 + + public init( + added: Set = [], + modified: Set = [], + deleted: Set = [], + sizeChange: Int64 = 0 + ) { + self.added = added + self.modified = modified + self.deleted = deleted + self.sizeChange = sizeChange + } + + /// Empty filesystem changes. + public static let empty = FilesystemChanges() + + /// Check if any changes were made. + public var hasChanges: Bool { + !added.isEmpty || !modified.isEmpty || !deleted.isEmpty + } +} diff --git a/Sources/NativeBuilder/README.md b/Sources/NativeBuilder/README.md new file mode 100644 index 00000000..4a9f13f2 --- /dev/null +++ b/Sources/NativeBuilder/README.md @@ -0,0 +1,52 @@ +# Swift Native Builder + +A pure-Swift container build system leveraging Containerization.framework for fast, secure, and reproducible builds. + +## Overview + +Swift Native Builder is a modern container build system that replaces traditional container builders with a Swift-native implementation. It runs each build step in isolated VMs, uses content-addressable storage for caching, and produces OCI-compliant container images. + +### Key Features + +- **Headless** - No daemon required, runs as a simple CLI tool +- **Mac-Native** - Built on Containerization.framework and Swift Concurrency +- **Secure** - Hardware-backed signing with Secure Enclave, biometric-protected secrets +- **Fast** - Parallel execution with intelligent caching + +## Quick Start + +```bash +# Build a container image +swift run builder build . + +# Build with secrets +swift run builder secret add github-token --biometric +swift run builder build --build-secret=id=github-token,target=/run/secrets/token . +``` + +## Architecture + +The project consists of several components: + +- **ContainerBuildIR** - Intermediate representation for build operations +- **Parser** - Dockerfile parser and build graph generator +- **Scheduler** - DAG scheduler for parallel execution +- **Executor** - VM-based execution engine +- **CAS** - Content-addressable storage for layers + +## Development + +```bash +# Build the project +swift build + +# Run tests +swift test + +# Run demo +swift run container-build-demo +``` + +## Status + +This project is under active development. The IR layer is implemented and functional. \ No newline at end of file diff --git a/Sources/NativeBuilder/docs/ContainerBuildCache/Architecture.md b/Sources/NativeBuilder/docs/ContainerBuildCache/Architecture.md new file mode 100644 index 00000000..10073cd9 --- /dev/null +++ b/Sources/NativeBuilder/docs/ContainerBuildCache/Architecture.md @@ -0,0 +1,313 @@ +# ContainerBuildCache Architecture + +This document outlines the high-level architecture of ContainerBuildCache, focusing on system design, component interaction, and data flow patterns. + +## Overview + +The cache is designed around three key layers: + +1. **BuildCache API Layer** - Public interface matching the BuildCache protocol +2. **Content-Based Cache Layer** - Manages cache entries as OCI artifacts with metadata indexing +3. **ContentStore** - Provides reliable, content-addressable storage with built-in deduplication + +### Design Principles + +- **Simplicity** - Clean separation of concerns with ContentStore handling storage complexity +- **Reliability** - Leverages ContentStore's atomic operations and content verification +- **Performance** - Content-addressable lookups with automatic deduplication and compression +- **Scalability** - Lightweight index with support for sharding and distributed storage +- **Maintainability** - Standard OCI artifact format with minimal custom code + +## High-Level Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ BuildCache API │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ get() │ │ put() │ │ statistics()│ │ +│ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ │ +│ │ │ │ │ +├─────────┴─────────────────┴─────────────────┴───────────────────┤ +│ BuildCache Implementation │ +│ ┌─────────────────────────────────────────────────────────┐ │ +│ │ Content-Based Cache Layer │ │ +│ │ ┌───────────┐ ┌──────────────┐ ┌────────────────┐ │ │ +│ │ │ Index │ │ Manifest │ │ Compressor │ │ │ +│ │ │ Manager │ │ Builder │ │ Engine │ │ │ +│ │ └─────┬─────┘ └──────┬───────┘ └────────┬───────┘ │ │ +│ │ │ │ │ │ │ +│ └────────┴────────────────┴────────────────────┴──────────┘ │ +│ │ │ +├────────────────────────────┴────────────────────────────────────┤ +│ ContentStore │ +│ ┌─────────────────────────────────────────────────────────┐ │ +│ │ Content-Addressable Storage (CAS) │ │ +│ │ - Deduplication │ │ +│ │ - Atomic Operations │ │ +│ │ - Content Verification │ │ +│ └─────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +## Component Responsibilities + +### 1. BuildCache API Layer +- **Purpose**: Provides the public interface matching the BuildCache protocol +- **Responsibilities**: + - Input validation and sanitization + - Error handling and user-friendly error messages + - API versioning and backward compatibility + - Metrics collection and logging + +### 2. Content-Based Cache Layer +- **Purpose**: Manages cache entries as OCI-compliant artifacts with metadata indexing +- **Components**: + - **Index Manager**: SQLite-based metadata storage for fast lookups + - **Manifest Builder**: Creates OCI-compliant manifests for cache entries + - **Compressor Engine**: Handles compression/decompression of cache data + +### 3. ContentStore +- **Purpose**: Provides reliable, content-addressable storage with built-in deduplication +- **Features**: + - Content-addressable storage (CAS) with SHA256 addressing + - Atomic operations ensuring consistency + - Built-in content verification and integrity checking + - Automatic deduplication of identical content + +## Data Flow Patterns + +### Cache PUT Operation Flow + +``` +Client BuildCache ContentStore Index + │ │ │ │ + ├─put(result, key)────>│ │ │ + │ │ │ │ + │ ├─1. Generate digest │ │ + │ │ from cache key │ │ + │ │ │ │ + │ ├─2. Serialize components │ │ + │ │ - Snapshot │ │ + │ │ - Environment │ │ + │ │ - Metadata │ │ + │ │ │ │ + │ ├─3. Store blobs─────────>│ │ + │ │<──────blob digests──────│ │ + │ │ │ │ + │ ├─4. Create manifest │ │ + │ │ │ │ + │ ├─5. Store manifest──────>│ │ + │ │<────manifest digest─────│ │ + │ │ │ │ + │ ├─6. Update index────────────────────────────>│ + │ │<───────────────────────────────success──────│ + │ │ │ │ + │<────────success──────│ │ │ +``` + +### Cache GET Operation Flow + +``` +Client BuildCache ContentStore Index + │ │ │ │ + ├─get(key)────────────>│ │ │ + │ │ │ │ + │ ├─1. Generate digest │ │ + │ │ from cache key │ │ + │ │ │ │ + │ ├─2. Lookup in index─────────────────────────>│ + │ │<───────────────────────entry metadata───────│ + │ │ │ │ + │ ├─3. Fetch manifest──────>│ │ + │ │<────manifest data───────│ │ + │ │ │ │ + │ ├─4. Fetch layers────────>│ │ + │ │<─────layer data─────────│ │ + │ │ │ │ + │ ├─5. Reconstruct result │ │ + │ │ │ │ + │ ├─6. Update access time──────────────────────>│ + │ │ │ │ + │<────CachedResult─────│ │ │ +``` +## Cache Key Generation + +Cache keys are deterministically generated from operation characteristics: + +``` +CacheKey = SHA256( + version || + operation_digest || + sorted(input_digests) || + normalized_platform || + operation_type || + operation_content +) +``` + +This ensures that identical operations with the same inputs produce the same cache key, enabling reliable cache hits across different build environments. + +## Cache Entry Architecture + +### OCI Artifact Structure + +Each cache entry follows the OCI artifact specification: + +``` +Cache Entry (OCI Artifact) +├── Manifest (JSON) +│ ├── schemaVersion: 2 +│ ├── mediaType: "application/vnd.container-build.cache.manifest.v2+json" +│ ├── config: CacheConfig +│ │ ├── cacheKey: SerializedCacheKey +│ │ ├── operationType: String +│ │ ├── platform: Platform +│ │ └── buildVersion: String +│ └── layers: [ +│ ├── Layer 1: Snapshot Data +│ │ ├── mediaType: "application/vnd.container-build.snapshot.v1+json" +│ │ ├── digest: "sha256:..." +│ │ └── size: Int64 +│ ├── Layer 2: Environment Changes (optional) +│ │ ├── mediaType: "application/vnd.container-build.environment.v1+json" +│ │ ├── digest: "sha256:..." +│ │ └── size: Int64 +│ └── Layer 3: Metadata (optional) +│ ├── mediaType: "application/vnd.container-build.metadata.v1+json" +│ ├── digest: "sha256:..." +│ └── size: Int64 +│ ] +└── Content Blobs + ├── Snapshot blob (compressed) + ├── Environment blob (if present) + └── Metadata blob (if present) +``` + +### Index Architecture + +The SQLite index provides fast metadata access: + +```sql +-- Cache entries table +CREATE TABLE cache_entries ( + digest TEXT PRIMARY KEY, + created_at INTEGER NOT NULL, + last_accessed_at INTEGER NOT NULL, + access_count INTEGER DEFAULT 1, + total_size INTEGER NOT NULL, + platform_os TEXT NOT NULL, + platform_arch TEXT NOT NULL, + operation_type TEXT NOT NULL +); + +-- Indexes for efficient queries +CREATE INDEX idx_lru ON cache_entries(last_accessed_at); +CREATE INDEX idx_age ON cache_entries(created_at); +CREATE INDEX idx_platform ON cache_entries(platform_os, platform_arch); +CREATE INDEX idx_size ON cache_entries(total_size); +``` + +## Eviction Architecture + +### Eviction Manager + +``` +┌─────────────────────────────────────────┐ +│ Eviction Manager │ +│ │ +│ 1. Check trigger conditions: │ +│ - Total size > maxSize │ +│ - Entry age > maxAge │ +│ - Manual trigger │ +│ │ +│ 2. Select victims: │ +│ - Query index by policy │ +│ - Build eviction list │ +│ │ +│ 3. Execute eviction: │ +│ - Remove from ContentStore │ +│ - Update index │ +│ - Log metrics │ +└─────────────────────────────────────────┘ +``` + +### Eviction Policies + +- **LRU (Least Recently Used)**: Evicts entries with oldest access time +- **LFU (Least Frequently Used)**: Evicts entries with lowest access count +- **FIFO (First In First Out)**: Evicts entries with oldest creation time +- **TTL (Time To Live)**: Evicts entries older than specified age +- **ARC (Adaptive Replacement Cache)**: Adaptive policy balancing recency and frequency + +## Concurrency Model + +### Actor-Based Design + +```swift +public actor ContentAddressableCache: BuildCache { + private let contentStore: ContentStore + private let index: CacheIndex + private let configuration: CacheConfiguration + + // All operations are serialized through the actor + public func get(_ key: CacheKey, for operation: ContainerBuildIR.Operation) async -> CachedResult? + public func put(_ result: CachedResult, key: CacheKey, for operation: ContainerBuildIR.Operation) async + public func statistics() async -> CacheStatistics +} +``` + +### Parallel Operations + +- **Layer Storage**: Multiple layers can be stored concurrently +- **Layer Retrieval**: Parallel fetching of cache entry layers +- **Background Cleanup**: Eviction runs in background tasks +- **Index Updates**: Batched for improved performance + +## Error Handling Strategy + +### Graceful Degradation + +1. **Index Corruption**: Rebuild from ContentStore manifests +2. **ContentStore Errors**: Fall back to cache miss behavior +3. **Partial Cache Entries**: Clean up orphaned data automatically +4. **Disk Space Issues**: Trigger aggressive eviction + +### Recovery Mechanisms + +- **Orphan Cleanup**: Remove index entries without corresponding ContentStore data +- **Consistency Checks**: Periodic validation of index vs ContentStore state +- **Automatic Repair**: Self-healing for common corruption scenarios + +## Performance Characteristics + +### Time Complexity + +- **Cache Lookup**: O(1) - Direct content-addressable access +- **Cache Storage**: O(1) - Parallel layer storage +- **Eviction Query**: O(log n) - Indexed database queries +- **Index Updates**: O(1) - Single row operations + +### Space Complexity + +- **Deduplication**: Automatic content deduplication in ContentStore +- **Compression**: Configurable compression levels for space/CPU tradeoff +- **Index Overhead**: Minimal metadata storage in SQLite + +This architecture provides a robust, scalable foundation for build caching while maintaining simplicity and leveraging proven storage technologies. + +## Benefits + +### Reliability +- **Atomic Operations** - ContentStore ensures crash-safe updates +- **Content Verification** - Built-in integrity checking prevents corruption +- **Deduplication** - Automatic space savings for identical content + +### Performance +- **O(1) Lookups** - Content-addressable storage enables fast retrieval +- **Parallel Operations** - Concurrent layer fetching and storage +- **Compression** - Reduces I/O overhead and storage requirements + +### Maintainability +- **Standard Format** - OCI artifacts are well-understood and toolable +- **Clear Data Model** - Explicit separation of concerns +- **Minimal Custom Code** - Leverages proven ContentStore implementation diff --git a/Sources/NativeBuilder/docs/ContainerBuildExecutor/Architecture.md b/Sources/NativeBuilder/docs/ContainerBuildExecutor/Architecture.md new file mode 100644 index 00000000..77637be6 --- /dev/null +++ b/Sources/NativeBuilder/docs/ContainerBuildExecutor/Architecture.md @@ -0,0 +1,177 @@ +# ContainerBuildExecutor Architecture + +## Overview + +The ContainerBuildExecutor implements a clean, production-ready execution layer for ContainerBuildIR using the **Executor Pattern** rather than a complex scheduler. + +## Architecture + +### Core Components + +1. **BuildExecutor** - The main orchestrator that executes complete build graphs +2. **OperationExecutor** - Executes individual operations (pluggable for different operation types) +3. **ExecutionContext** - Carries mutable state through the build process +4. **ExecutionDispatcher** - Routes operations to appropriate executors based on capabilities +5. **BuildCache** - Caches operation results to avoid redundant work +6. **Snapshotter** - Manages filesystem snapshots for layer creation + +### Design Principles + +- **Simplicity** - Clean interfaces with minimal complexity +- **Extensibility** - Easy to add new operation types and executors +- **Performance** - Parallel execution where possible, with efficient caching +- **Type Safety** - Leverages Swift's type system for correctness + +## Execution Flow + +```mermaid +graph TD + A[BuildGraph] --> B[SimpleExecutor] + B --> C[Stage Ordering] + C --> D[For Each Stage] + D --> E[Topological Sort] + E --> F[For Each Node] + F --> G{Check Cache} + G -->|Hit| H[Return Cached Result] + G -->|Miss| I[ExecutionDispatcher] + I --> J[Find Suitable Executor] + J --> K[Execute Operation] + K --> L[Update Context] + L --> M[Cache Result] + M --> N[Next Node] +``` + +## Capability-Based Routing + +The dispatcher matches operations to executors based on: + +1. **Operation Type** - Does the executor support this operation kind? +2. **Platform** - Can the executor handle the target platform? +3. **Privileges** - Does the executor have required privileges? +4. **Resources** - Are resource requirements satisfied? + +## Concurrency Model + +### Stage-Level Execution + +Stages are executed sequentially to respect dependencies: + +```swift +for stage in graph.stagesInDependencyOrder() { + let context = ExecutionContext(stage: stage, ...) + let snapshot = try await executeStage(stage, context: context) + stageSnapshots[stage.name] = snapshot +} +``` + +### Node-Level Parallelism + +Within a stage, independent nodes execute concurrently: + +```swift +let levels = try GraphTraversal.topologicalLevels(stage) +for level in levels { + await withTaskGroup(of: ExecutionResult.self) { group in + for node in level { + group.addTask { + try await executeNode(node, context) + } + } + } +} +``` + +## State Management + +### Snapshot Evolution + +Each operation produces a new filesystem snapshot: + +``` +Initial Snapshot (S0) + ↓ +Operation 1 → Snapshot S1 + ↓ +Operation 2 → Snapshot S2 + ↓ +Operation 3 → Snapshot S3 (Final) +``` + +### Environment Propagation + +Environment changes cascade through operations: + +```swift +context.updateEnvironment(["FOO": .literal("bar")]) +// This update is visible to all subsequent operations in the stage +``` + +## Caching Strategy + +### Cache Key Generation + +Cache keys include: +- Operation digest (content hash) +- Input digests (from dependencies) +- Platform identifier +- Additional context + +### Cache Lookup Flow + +1. Compute cache key for operation +2. Check cache for existing result +3. If hit: skip execution, use cached result +4. If miss: execute operation, store result + +## Error Handling + +Errors are categorized and handled appropriately: + +- **Unsupported Operations** - No executor can handle the operation +- **Resource Constraints** - Requirements cannot be satisfied +- **Execution Failures** - Operation failed during execution +- **Cancellation** - Build was cancelled by user + +## Extensibility Points + +### Adding New Operation Types + +1. Define the operation in ContainerBuildIR +2. Create a specific executor implementing `OperationExecutor` +3. Register the executor with the dispatcher + +### Custom Caching + +Implement the `BuildCache` protocol: + +```swift +public protocol BuildCache: Sendable { + func get(_ key: CacheKey, for operation: Operation) async -> CachedResult? + func put(_ result: ExecutionResult, key: CacheKey, for operation: Operation) async +} +``` + +### Alternative Snapshotters + +Implement the `Snapshotter` protocol for different backends: + +```swift +public protocol Snapshotter: Sendable { + func createSnapshot(from parent: Snapshot?, applying changes: FilesystemChanges) async throws -> Snapshot + func prepare(_ snapshot: Snapshot) async throws -> SnapshotHandle +} +``` + +## Performance Considerations + +1. **Lazy Snapshot Creation** - Only create snapshots when filesystem changes occur +2. **Parallel Execution** - Maximize concurrency within dependency constraints +3. **Efficient Caching** - Cache keys designed for fast lookup +4. **Resource Pooling** - Reuse expensive resources like container instances + +## Future Enhancements + +1. **Distributed Execution** - Execute operations across multiple machines +2. **Incremental Builds** - Skip unchanged portions of the graph +3. **Progress Reporting** - Real-time feedback during execution +4. **Resource Monitoring** - Track CPU, memory, and I/O usage \ No newline at end of file diff --git a/Sources/NativeBuilder/docs/ContainerBuildIR/Graph.md b/Sources/NativeBuilder/docs/ContainerBuildIR/Graph.md new file mode 100644 index 00000000..9a512b75 --- /dev/null +++ b/Sources/NativeBuilder/docs/ContainerBuildIR/Graph.md @@ -0,0 +1,238 @@ +# Build Graph Architecture + +The ContainerBuildIR build graph is a directed acyclic graph (DAG) that represents the sequence of operations needed to build a container image. This document explains the design decisions, tradeoffs, and implementation details. + +## Design Overview + +### Core Structure + +```swift +BuildGraph +├── stages: [BuildStage] +├── targetStage: BuildStage? +├── buildArgs: [String: BuildArg] +└── targetPlatforms: [Platform] + +BuildStage +├── name: String? +├── base: ImageOperation +├── nodes: [BuildNode] +└── platform: Platform? + +BuildNode +├── id: UUID +├── operation: Operation +└── dependencies: Set +``` + +### Design Rationale + +#### 1. Stage-Based Organization + +**Why**: Container builds naturally organize into stages (multi-stage builds), where each stage can: +- Start from a different base image +- Be referenced by other stages +- Produce intermediate artifacts + +**Tradeoff**: Adds complexity compared to a flat operation list, but enables: +- Clear separation of build phases +- Efficient layer caching strategies +- Support for `COPY --from` patterns + +#### 2. UUID-Based Node Identity + +**Why**: Using UUIDs for node identification provides: +- Guaranteed uniqueness without coordination +- Stable references across graph transformations +- No naming conflicts + +**Tradeoff**: Less human-readable than string names, but ensures correctness in complex graphs. + +#### 3. Explicit Dependencies + +**Why**: Each node explicitly declares its dependencies rather than relying on insertion order: +- Enables parallel execution of independent operations +- Makes the graph self-documenting +- Simplifies graph analysis and optimization + +**Tradeoff**: Requires explicit dependency management, but prevents implicit ordering bugs. + +## Graph Construction + +### Using GraphBuilder + +The `GraphBuilder` provides a fluent API for constructing graphs: + +```swift +// Single-stage build +let graph = try GraphBuilder.singleStage( + from: ImageReference(parsing: "ubuntu:22.04")!, + platform: .linuxAMD64 +) { builder in + builder + .run("apt-get update") + .run("apt-get install -y python3") + .workdir("/app") + .copyFromContext(["*.py"], to: "/app/") + .cmd(Command.exec(["python3", "app.py"])) +} +``` + +## Dependency Management + +### Automatic Dependencies + +The GraphBuilder automatically manages dependencies based on operation order: + +```swift +builder + .run("command1") // No dependencies + .run("command2") // Depends on command1 + .run("command3") // Depends on command2 +``` + +### Cross-Stage Dependencies + +Dependencies between stages are tracked through stage references: + +```swift +// This creates an implicit dependency on the "builder" stage +.copyFromStage(.named("builder"), paths: ["/app"], to: "/") +``` + +### Parallel Operations + +Operations without dependencies can execute in parallel: + +```swift +// These operations have no interdependencies +let node1 = BuildNode(operation: op1, dependencies: []) +let node2 = BuildNode(operation: op2, dependencies: []) +let node3 = BuildNode(operation: op3, dependencies: [node1.id, node2.id]) +// node1 and node2 can run in parallel, node3 waits for both +``` + +## Graph Analysis + +### Traversal Utilities + +The framework provides utilities for graph analysis: + +```swift +// Topological sort for execution order +let executionOrder = try GraphTraversal.topologicalSort(stage) + +// Find entry points (nodes with no dependencies) +let roots = GraphTraversal.findRoots(in: stage) + +// Find terminal nodes +let leaves = GraphTraversal.findLeaves(in: stage) + +// Check for cycles +GraphTraversal.detectCycles(in: stage) // Throws if cycles exist +``` + +### Visitor Pattern + +Use the visitor pattern to analyze or transform the graph: + +```swift +class DependencyAnalyzer: OperationVisitor { + private var packageCommands: [String] = [] + + func visit(_ operation: ExecOperation) { + if case .shell(let cmd) = operation.command, + cmd.contains("apt-get install") || cmd.contains("pip install") { + packageCommands.append(cmd) + } + } +} + +// Apply visitor to all operations +let analyzer = DependencyAnalyzer() +for stage in graph.stages { + for node in stage.nodes { + node.operation.accept(analyzer) + } +} +``` + +## Best Practices + +### 1. Keep Stages Focused + +Each stage should have a single responsibility: +- Dependencies stage +- Build stage +- Runtime stage + +### 2. Minimize Inter-Stage Dependencies + +Reduce coupling between stages by only copying necessary artifacts: + +```swift +// Good: Copy only the binary +.copyFromStage(.named("builder"), paths: ["/app/binary"], to: "/usr/local/bin/") + +// Avoid: Copying entire directories unnecessarily +.copyFromStage(.named("builder"), paths: ["/"], to: "/") +``` + +### 3. Use Platform-Specific Stages + +When building for multiple platforms: + +```swift +let graph = BuildGraph( + stages: stages, + targetPlatforms: [.linuxAMD64, .linuxARM64] +) +``` + +### 4. Leverage Validation + +Always validate graphs before execution: + +```swift +let validator = StandardValidator() +let result = validator.validate(graph) +if !result.isValid { + // Handle validation errors +} +``` + +## Performance Considerations + +### Memory Usage + +- Graphs are immutable after construction +- Node operations are copy-on-write +- Large graphs (1000+ nodes) use ~100KB of memory + +### Construction Performance + +- GraphBuilder uses efficient array building +- O(1) node insertion +- O(n) validation where n is node count + +### Traversal Performance + +- Topological sort: O(V + E) where V is vertices, E is edges +- Cycle detection: O(V + E) +- Visitor traversal: O(V) + +## Future Considerations + +### Potential Enhancements + +1. **Subgraph Extraction**: Extract portions of the graph for partial builds +2. **Graph Merging**: Combine multiple graphs for complex workflows +3. **Lazy Evaluation**: Defer operation construction until needed +4. **Graph Caching**: Serialize graphs for faster subsequent loads + +### Maintaining Compatibility + +The graph structure is designed for extensibility: +- New operation types can be added without breaking existing graphs +- Additional metadata can be attached to nodes +- Stage properties can be extended \ No newline at end of file diff --git a/Sources/NativeBuilder/docs/ContainerBuildIR/Operations.md b/Sources/NativeBuilder/docs/ContainerBuildIR/Operations.md new file mode 100644 index 00000000..7ff36ec6 --- /dev/null +++ b/Sources/NativeBuilder/docs/ContainerBuildIR/Operations.md @@ -0,0 +1,231 @@ +# Operations Design + +Operations are the fundamental building blocks of the ContainerBuildIR. This document explains their design, implementation patterns, and the rationale behind key decisions. + +## Design Overview + +### Operation Protocol + +```swift +public protocol Operation: Sendable { + /// Unique type identifier + static var operationKind: OperationKind { get } + + /// Instance operation kind + var operationKind: OperationKind { get } + + /// Accept a visitor for traversal + func accept(_ visitor: V) throws -> V.Result +} + +// Operations also conform to Codable, Hashable, and Equatable +// through protocol extensions or direct conformance +``` + +### Core Operation Types + +1. **ExecOperation** - Command execution (RUN) +2. **FilesystemOperation** - File manipulation (COPY, ADD) +3. **ImageOperation** - Base image specification (FROM) +4. **MetadataOperation** - Container metadata (ENV, LABEL, USER) + +## Design Philosophy + +### 1. Protocol-Based Design + +**Why**: Using protocols instead of enums provides: +- Open extensibility for custom operations +- Type safety with associated types +- Clean separation of concerns + +**Tradeoff**: Requires visitor pattern for exhaustive handling, but enables third-party extensions. + +### 2. Immutable Operations + +**Why**: All operations are immutable value types: +- Thread-safe by default (Sendable) +- Predictable behavior +- Easy to reason about + +**Tradeoff**: Modifications require creating new instances, but prevents accidental mutations. + +### 3. Self-Contained Operations + +**Why**: Each operation contains all information needed for execution: +- No external state dependencies +- Simplifies serialization +- Enables operation reuse + +**Tradeoff**: Some data duplication possible, but ensures operation independence. + +## Implementing Custom Operations + +### Step 1: Define the Operation + +```swift +public struct CompressOperation: Operation, Codable, Hashable { + public let sourcePath: String + public let algorithm: CompressionAlgorithm + public let level: Int + public let metadata: OperationMetadata? + + public static let operationKind = OperationKind(rawValue: "compress") + public var operationKind: OperationKind { Self.operationKind } + + public func accept(_ visitor: V) throws -> V.Result { + // Custom operations use visitUnknown + return try visitor.visitUnknown(self) + } +} + +public enum CompressionAlgorithm: String, Codable, Sendable { + case gzip + case bzip2 + case xz + case zstd +} +``` + +### Step 2: Visitor Pattern + +The OperationVisitor protocol provides methods for all built-in operations: + +```swift +public protocol OperationVisitor { + associatedtype Result + + func visit(_ operation: ExecOperation) throws -> Result + func visit(_ operation: FilesystemOperation) throws -> Result + func visit(_ operation: ImageOperation) throws -> Result + func visit(_ operation: MetadataOperation) throws -> Result + func visitUnknown(_ operation: Operation) throws -> Result +} +``` + +Custom operations are handled through `visitUnknown`, which provides a default implementation that throws an error for unrecognized operations. + +### Step 3: Operation Metadata + +All operations can include metadata for debugging and analysis: + +```swift +public struct OperationMetadata: Codable, Hashable, Sendable { + public let description: String? + public let location: SourceLocation? + public let annotations: [String: String]? + public let cacheConfig: CacheConfig? +} + +public struct SourceLocation: Codable, Hashable, Sendable { + public let file: String? + public let line: Int? + public let column: Int? +} +``` + +### Step 4: Add Builder Support + +```swift +extension StageBuilder { + @discardableResult + public func compress(_ path: String, algorithm: CompressionAlgorithm = .gzip, level: Int = 6) -> Self { + let operation = CompressOperation( + sourcePath: path, + algorithm: algorithm, + level: level + ) + addNode(BuildNode(operation: operation)) + return self + } +} +``` + +## Performance Considerations + +### Memory Efficiency + +Operations are designed to be lightweight: +- Use copy-on-write for collections +- Share common data through references +- Typical operation: 200-500 bytes + +### Serialization Performance + +- Codable implementation is optimized for speed +- Custom operations should implement efficient coding +- Consider using CodingKeys for stable serialization + +## Best Practices + +### 1. Keep Operations Focused + +Each operation should do one thing well: +```swift +// Good: Single responsibility +ExecOperation(command: .shell("apt-get update")) +ExecOperation(command: .shell("apt-get install -y curl")) + +// Avoid: Multiple unrelated commands +ExecOperation(command: .shell("apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*")) +``` + +### 2. Use Type-Safe Enums + +Prefer enums over strings for operation parameters: +```swift +// Good: Type-safe +public enum PackageManager { + case apt + case yum + case apk +} + +// Avoid: Stringly-typed +let packageManager = "apt-get" +``` + +### 3. Provide Meaningful Descriptions + +Implement descriptive `description` properties: +```swift +public var description: String { + switch action { + case .copy: + return "Copy \(source.displayName) to \(destination)" + case .add: + return "Add \(source.displayName) to \(destination)" + case .remove: + return "Remove \(destination)" + } +} +``` + +### 4. Design for Extensibility + +Consider future needs when designing operations: +```swift +public struct ExecOperation: Operation { + // Core functionality + public let command: Command + public let environment: Environment + + // Extensibility points + public let metadata: [String: Any]? // For future extensions + public let extensions: OperationExtensions? // Type-safe extensions +} +``` + +## Future Directions + +### Potential Enhancements + +1. **Operation Macros**: Higher-level operations that expand to multiple primitives +2. **Conditional Operations**: Operations that execute based on runtime conditions +3. **Parallel Operations**: Explicit parallel execution hints +4. **Operation Fragments**: Reusable operation templates + +### Maintaining Backward Compatibility + +- New operation types can be added without breaking existing code +- Optional properties can be added to existing operations +- The visitor pattern allows graceful handling of unknown operations \ No newline at end of file diff --git a/Sources/NativeBuilder/docs/Design.md b/Sources/NativeBuilder/docs/Design.md new file mode 100644 index 00000000..bc03bf95 --- /dev/null +++ b/Sources/NativeBuilder/docs/Design.md @@ -0,0 +1,238 @@ +Swift Native Builder + +## Introduction + +Swift Native Builder is a pure-Swift container build system that replaces the current Swift+Go architecture. It leverages Containerization.framework to run each build step in isolated VMs, writing output to a content-addressable store, for faster, deterministic and reproducible builds while maintaining native performance. + +### Design Principles + +- **Headless**: No resident daemon. CLI spins up, orchestrates the build, and exits cleanly +- **Mac-Native**: Native integration with Containerization.framework, Swift Concurrency, Swift Data, Keychain +- **Secure by default**: Hardware-backed signing (Secure Enclave), biometric authentication, opt-in secrets via `--build-secret` +- **Fast**: Maximal parallelism with incremental caching and negligible per-step overhead +- **OCI-compliant**: Produces standard container images without external dependencies + +## High-Level Architecture +``` +┌─────────────────────────┐ +│ builder build │ +└───────────┬─────────────┘ + │ + ▼ +┌─────────────────────────┐ +│ Parser │ +└───────────┬─────────────┘ + │ + ▼ +┌─────────────────────────┐ +│ DAG Scheduler │ +└───────────┬─────────────┘ + │ + │ + ▼ ┌──────────────────┐ +┌────────────────────────┐ │ │ +│ Executor │◄─────────────| Content │ +│ VM-RUN │ | Addressable │ +└────────────┬───────────┘ │ Store │ + | │ (CAS) │ + | │ │ + │ └────────▲─────────┘ + ▼ │ +┌─────────────────────────┐ │ +│ Diff & Snapshotter ├──────────────────────┘ +└────────────┬────────────┘ + │ + ▼ + ┌─────────┐ + │ Signer │ + └────┬────┘ + │ + ▼ + ┌────────────────┐ + │ OCI Image / │ + │ ext4 block │ + └────────────────┘ +``` + +### Parser + +The parser must be **fully Dockerfile-compliant**, supporting all current semantics. More importantly, the parser architecture must **accommodate future evolution** without breaking changes. + +#### Intermediate Representation (IR) + +The core of the parser is an **extensible IR** that can represent any container build operation (just like buildkit's LLB). + +See [**ContainerBuildIR**](./ContainerBuildIR/) for more detailed information. + +This IR is intentionally generic - it represents operations, not Dockerfile instructions. This allows: +- Any Dockerfile construct to map to these primitives +- Future instructions to reuse existing operations +- Non-Dockerfile frontends to target the same IR + +#### Frontend + +A **Frontend** transforms a Dockerfile into our IR: + +```swift +protocol Frontend { + associatedtype InputFormat // Dockerfile or other format + func transform(_ input: InputFormat) throws -> BuildGraph +} + +// Dockerfile frontend implementation +struct DockerfileFrontend: Frontend { + typealias InputFormat = Dockerfile + + func transform(_ dockerfile: Dockerfile) throws -> BuildGraph { + // Handles all Dockerfile semantics: + // - Stage name resolution for COPY --from + // - Build arg substitution + // - Multi-stage dependency tracking + // - Cache mount specifications + } +} +``` + +This separation enables: +- Multiple frontend languages (Dockerfile, Buildkit LLB, future formats) +- Frontend-specific optimizations without affecting the build engine +- Easier testing of language semantics vs execution semantics + +#### Build Graph + +The parser's only job is to produce a valid AST. All semantic understanding (stage references, variable substitution, etc.) happens in the construction of the Build Graph. + +The Build Graph is a directed acyclic graph (DAG) of operations, where each node represents an operation and each edge represents a dependency between operations. This clean separation ensures we can evolve the language without touching the core build engine. + + +See [**ContainerBuildIR**](./ContainerBuildIR/) for more detailed information. + +### Scheduler + +The Scheduler orchestrates the execution of the build graph once it's fully constructed by the parser. It analyzes the dependency graph to maximize parallelism, executing all nodes whose input artifacts are available while respecting the topological ordering of dependencies. + +#### Core Components + +A complete build operation requires three components (Scheduler, Executor, Cache) working in concert. + +#### Execution Flow + +1. **Initialization**: Scheduler receives the complete BuildGraph from the parser +2. **Dependency Analysis**: Identifies nodes with no dependencies (typically base image pulls) +3. **Parallel Execution**: Launches concurrent tasks for all executable nodes +4. **Artifact Storage**: Each completed node's output is immediately written to the CAS +5. **Progress Tracking**: As nodes complete, their artifacts unlock dependent nodes +6. **Completion**: Returns when all nodes have executed successfully + +#### Cache Integration + +The scheduler treats the CAS as the source of truth for artifacts. + +This architecture enables: +- Maximum parallelism within dependency constraints +- Cache-aware execution to skip redundant work +- Clean separation between scheduling logic and execution mechanics +- Support for speculative execution + +### Executor Registry + +The Executor Registry maintains a collection of executors, each advertising specific capabilities. For any given operation and its constraints, the registry selects the first matching executor. + +#### Built-in Executors + +| Executor | Operations | Description | +|----------|------------|-------------| +| `VMExecutor` | `RUN`, `SHELL` | Spins up LinuxKit VM, mounts parent layer via virtio-fs | +| `NativeExecutor` | `COPY`, `ADD` | Direct filesystem operations with copy-on-write | +| `CacheExecutor` | `--mount=type=cache` | Manages persistent cache directories | +| `MetadataExecutor` | `ENV`, `LABEL`, `ARG` | Updates image config without execution | + +The registry uses first-match selection, allowing specialized executors (WASM, GPU) to take precedence over general-purpose ones. + +### Differ & Snapshotter + +The Differ and Snapshotter work together to capture filesystem changes after each build step, converting them into portable OCI layers. + +#### Differ + +The Differ computes the delta between filesystem states before and after an operation: + +#### Snapshotter + +The Snapshotter manages filesystem snapshots and converts layers to OCI format: + +#### Layer Format + +Each layer follows the OCI Image Layer Specification: + +``` +layer.tar.gz +├── bin/ +│ └── myapp # Added file +├── etc/ +│ └── config.json # Modified file +├── .wh.oldfile # Deletion marker +└── .wh..wh..opq # Opaque directory marker +``` + +This architecture enables: +- Efficient storage through deduplication +- Fast layer generation using native filesystem features +- Full OCI compatibility for cross-platform deployment +- Minimal overhead for unchanged files + + +### Content-Addressable Store (CAS) + +The CAS serves as the central artifact repository, storing all build outputs indexed by content hash. Its design mirrors containerd's content store for compatibility. + +#### Storage Layout + +``` +content/ +├── sha256/ +│ ├── 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a +│ ├── 6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b +│ └── ... +└── metadata/ + └── ... +``` + +#### Pluggable Backends + +The protocol enables multiple storage implementations: +- `LocalCache`: Filesystem-based storage with atomic writes +- `S3Cache`: Remote storage for distributed builds +- `HybridCache`: Tiered storage with local fast path + +### Security + +Swift Native Builder leverages macOS security features to protect secrets and ensure image integrity. + +#### Secret Management + +Build secrets are stored in the system Keychain with hardware-backed encryption. Secrets are injected at build time without persisting in layers. + +```bash +# Use in build without exposing in image +builder build --build-secret=id=github-token,target=/run/secrets/token . +``` + +#### Image Signing + +Every built image is cryptographically signed using the Secure Enclave: + +#### Trust Verification + +Images are verified before execution. + +This architecture provides: +- Zero-trust secret management with biometric protection +- Hardware-backed image signatures +- Transparent verification without runtime overhead +- Compatibility with existing OCI signature specifications + +### Future Work + +* SBOM and provenance generation +* Support for additional image formats (e.g. OCI, Docker, etc.) diff --git a/Sources/NativeBuilder/docs/README.md b/Sources/NativeBuilder/docs/README.md new file mode 100644 index 00000000..0d0a02dd --- /dev/null +++ b/Sources/NativeBuilder/docs/README.md @@ -0,0 +1,39 @@ +# Swift Native Builder Documentation + +Welcome to the Swift Native Builder documentation. These docs contain detailed information about the project's architecture and design decisions. + +## Documentation Structure + +### Core Documentation + +- [**Design.md**](./Design.md) - Complete architectural design and implementation details + - High-level architecture overview + - Component descriptions (Parser, Scheduler, Executor, CAS) + - Security model and extensibility + +### Component Documentation + +- [**ContainerBuildIR**](./ContainerBuildIR/) - Intermediate Representation documentation + - Core types and operations + - Build graph structure + - Validation and analysis tools + +- [**ContainerBuildExecutor**](./ContainerBuildExecutor/) - Execution layer documentation + - Executor architecture and patterns + +- [**ContainerBuildCache**](./ContainerBuildCache/) - Caching layer documentation + - ContentStore-based cache architecture + - OCI-compliant cache entry format + - Eviction policies and index management + +## Quick Links + +- [Project README](../README.md) - Getting started and overview +- [Examples](../Sources/ContainerBuildDemo/) - Sample code and usage patterns + +## Contributing + +When adding new documentation: +1. Place architectural documents in this docs folder +2. Component-specific documentation goes under `docs//` +3. Update this README with links to new documents \ No newline at end of file diff --git a/Tests/NativeBuilderTests/ContainerBuildCacheTests/BuildCacheProtocolTests.swift b/Tests/NativeBuilderTests/ContainerBuildCacheTests/BuildCacheProtocolTests.swift new file mode 100644 index 00000000..d10deef7 --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildCacheTests/BuildCacheProtocolTests.swift @@ -0,0 +1,220 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildSnapshotter +import ContainerizationOCI +import Foundation +import Testing + +@testable import ContainerBuildCache + +struct BuildCacheProtocolTests { + + // MARK: - MemoryBuildCache Tests + + @Test func memoryBuildCacheBasicOperations() async throws { + let cache = MemoryBuildCache() + + // Create test data + let key = createTestCacheKey() + let result = createTestCachedResult() + let operation = createTestOperation() + + // Test cache miss + let initialResult = await cache.get(key, for: operation) + #expect(initialResult == nil) + + // Test put + await cache.put(result, key: key, for: operation) + + // Test cache hit + let cachedResult = await cache.get(key, for: operation) + #expect(cachedResult != nil) + #expect(cachedResult?.snapshot.id == result.snapshot.id) + #expect(cachedResult?.environmentChanges.count == result.environmentChanges.count) + #expect(cachedResult?.metadataChanges == result.metadataChanges) + } + + @Test func memoryBuildCacheStatistics() async throws { + let cache = MemoryBuildCache() + + // Initial statistics + let initialStats = await cache.statistics() + #expect(initialStats.entryCount == 0) + #expect(initialStats.totalSize == 0) + #expect(initialStats.hitRate == 0) + + // Add some entries + let key1 = createTestCacheKey(operationDigest: "sha256:1111111111111111111111111111111111111111111111111111111111111111") + let key2 = createTestCacheKey(operationDigest: "sha256:2222222222222222222222222222222222222222222222222222222222222222") + let result = createTestCachedResult() + let operation = createTestOperation() + + await cache.put(result, key: key1, for: operation) + await cache.put(result, key: key2, for: operation) + + // Test statistics after adding entries + let statsAfterPut = await cache.statistics() + #expect(statsAfterPut.entryCount == 2) + #expect(statsAfterPut.totalSize > 0) + + // Test hit rate calculation + _ = await cache.get(key1, for: operation) // Hit + _ = await cache.get(key1, for: operation) // Hit + let nonExistentKey = createTestCacheKey(operationDigest: "sha256:9999999999999999999999999999999999999999999999999999999999999999") + _ = await cache.get(nonExistentKey, for: operation) // Miss + + let finalStats = await cache.statistics() + #expect(abs(finalStats.hitRate - 2.0 / 3.0) < 0.01) // 2 hits out of 3 attempts + } + + @Test func memoryBuildCacheConcurrentAccess() async throws { + let cache = MemoryBuildCache() + let operation = createTestOperation() + + // Test concurrent puts and gets + await withTaskGroup(of: Void.self) { group in + // Concurrent puts + for i in 0..<10 { + let index = i + let key = createTestCacheKey(operationDigest: "sha256:\(String(format: "%064d", index))") + let result = createTestCachedResult() + group.addTask { @Sendable in + await cache.put(result, key: key, for: operation) + } + } + + // Concurrent gets + for i in 0..<10 { + let index = i + let key = createTestCacheKey(operationDigest: "sha256:\(String(format: "%064d", index))") + group.addTask { @Sendable in + _ = await cache.get(key, for: operation) + } + } + } + + let stats = await cache.statistics() + #expect(stats.entryCount == 10) + } + + // MARK: - NoOpBuildCache Tests + + @Test func noOpBuildCacheAlwaysReturnsNil() async throws { + let cache = NoOpBuildCache() + let key = createTestCacheKey() + let result = createTestCachedResult() + let operation = createTestOperation() + + // Test that get always returns nil + let initialResult = await cache.get(key, for: operation) + #expect(initialResult == nil) + + // Test that put doesn't store anything + await cache.put(result, key: key, for: operation) + + // Test that get still returns nil after put + let afterPutResult = await cache.get(key, for: operation) + #expect(afterPutResult == nil) + } + + @Test func noOpBuildCacheStatistics() async throws { + let cache = NoOpBuildCache() + + let stats = await cache.statistics() + #expect(stats.entryCount == 0) + #expect(stats.totalSize == 0) + #expect(stats.hitRate == 0) + #expect(stats.oldestEntryAge == 0) + #expect(stats.mostRecentEntryAge == 0) + } + + // MARK: - CacheKey Tests + + @Test func cacheKeyEquality() throws { + let digest1 = try Digest(parsing: "sha256:1111111111111111111111111111111111111111111111111111111111111111") + let digest2 = try Digest(parsing: "sha256:2222222222222222222222222222222222222222222222222222222222222222") + let platform = Platform.linuxAMD64 + + let key1 = CacheKey(operationDigest: digest1, inputDigests: [digest2], platform: platform) + let key2 = CacheKey(operationDigest: digest1, inputDigests: [digest2], platform: platform) + let key3 = CacheKey(operationDigest: digest2, inputDigests: [digest1], platform: platform) + + #expect(key1 == key2) + #expect(key1 != key3) + } + + @Test func cacheKeyHashing() throws { + let digest1 = try Digest(parsing: "sha256:1111111111111111111111111111111111111111111111111111111111111111") + let digest2 = try Digest(parsing: "sha256:2222222222222222222222222222222222222222222222222222222222222222") + let platform = Platform.linuxAMD64 + + let key1 = CacheKey(operationDigest: digest1, inputDigests: [digest2], platform: platform) + let key2 = CacheKey(operationDigest: digest1, inputDigests: [digest2], platform: platform) + + #expect(key1.hashValue == key2.hashValue) + + // Test that keys can be used in sets + let keySet: Set = [key1, key2] + #expect(keySet.count == 1) // Should deduplicate + } + + // MARK: - CachedResult Tests + + @Test func cachedResultInitialization() throws { + let snapshot = createTestSnapshot() + let environmentChanges: [String: EnvironmentValue] = [ + "PATH": .literal("/usr/bin"), + "HOME": .literal("/home/user"), + ] + let metadataChanges = ["build.time": "2024-01-01T12:00:00Z"] + + let result = CachedResult( + snapshot: snapshot, + environmentChanges: environmentChanges, + metadataChanges: metadataChanges + ) + + #expect(result.snapshot.id == snapshot.id) + #expect(result.environmentChanges.count == 2) + #expect(result.metadataChanges["build.time"] == "2024-01-01T12:00:00Z") + } + + // MARK: - Helper Methods + + private func createTestCacheKey(operationDigest: String = "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef") -> ContainerBuildCache.CacheKey { + let digest = try! Digest(parsing: operationDigest) + let inputDigest = try! Digest(parsing: "sha256:fedcba0987654321fedcba0987654321fedcba0987654321fedcba0987654321") + return ContainerBuildCache.CacheKey( + operationDigest: digest, + inputDigests: [inputDigest], + platform: Platform.linuxAMD64 + ) + } + + private func createTestSnapshot() -> Snapshot { + TestDataFactory.createSnapshot() + } + + private func createTestCachedResult() -> CachedResult { + TestDataFactory.createCachedResult() + } + + private func createTestOperation() -> MockOperation { + TestDataFactory.createOperation() + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildCacheTests/CacheConfigurationTests.swift b/Tests/NativeBuilderTests/ContainerBuildCacheTests/CacheConfigurationTests.swift new file mode 100644 index 00000000..0ab111bd --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildCacheTests/CacheConfigurationTests.swift @@ -0,0 +1,332 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import Foundation +import Testing + +@testable import ContainerBuildCache + +struct CacheConfigurationTests { + + // MARK: - CacheConfiguration Tests + + @Test func cacheConfigurationDefaultValues() throws { + let config = CacheConfiguration() + + #expect(config.maxSize == 10 * 1024 * 1024 * 1024) // 10GB + #expect(config.maxAge == 7 * 24 * 60 * 60) // 7 days + #expect(config.evictionPolicy == .lru) + #expect(config.compression.algorithm == .zstd) + #expect(config.compression.level == 3) + #expect(config.compression.minSize == 1024) + #expect(config.verifyIntegrity == true) + #expect(config.sharding == nil) + #expect(config.gcInterval == 3600) // 1 hour + #expect(config.cacheKeyVersion == "v1") + #expect(config.defaultTTL == nil) + } + + @Test func cacheConfigurationCustomValues() throws { + let customIndexPath = FileManager.default.temporaryDirectory + .appendingPathComponent("custom-cache.db") + + let customCompression = CompressionConfiguration( + algorithm: .gzip, + level: 6, + minSize: 2048 + ) + + let customConcurrency = ConcurrencyConfiguration( + maxConcurrentReads: 50, + maxConcurrentWrites: 5, + maxConcurrentEvictions: 1 + ) + + let config = CacheConfiguration( + maxSize: 5 * 1024 * 1024 * 1024, // 5GB + maxAge: 3 * 24 * 60 * 60, // 3 days + compression: customCompression, + indexPath: customIndexPath, + evictionPolicy: .fifo, + concurrency: customConcurrency, + verifyIntegrity: false, + sharding: nil, + gcInterval: 1800, // 30 minutes + cacheKeyVersion: "v2", + defaultTTL: 86400 // 1 day + ) + + #expect(config.maxSize == 5 * 1024 * 1024 * 1024) + #expect(config.maxAge == 3 * 24 * 60 * 60) + #expect(config.compression.algorithm == .gzip) + #expect(config.compression.level == 6) + #expect(config.compression.minSize == 2048) + #expect(config.indexPath == customIndexPath) + #expect(config.evictionPolicy == .fifo) + #expect(config.concurrency.maxConcurrentReads == 50) + #expect(config.concurrency.maxConcurrentWrites == 5) + #expect(config.concurrency.maxConcurrentEvictions == 1) + #expect(config.verifyIntegrity == false) + #expect(config.sharding == nil) + #expect(config.gcInterval == 1800) + #expect(config.cacheKeyVersion == "v2") + #expect(config.defaultTTL == 86400) + } + + @Test func cacheConfigurationValidationLimits() throws { + // Test that configuration accepts reasonable limits + let config = CacheConfiguration( + maxSize: 1024, // 1KB minimum + maxAge: 60, // 1 minute minimum + gcInterval: 30 // 30 seconds minimum + ) + + #expect(config.maxSize == 1024) + #expect(config.maxAge == 60) + #expect(config.gcInterval == 30) + } + + // MARK: - EvictionPolicy Tests + + @Test func evictionPolicyLRU() throws { + let policy = EvictionPolicy.lru + #expect(policy == .lru) + + // Test that LRU is the default + let config = CacheConfiguration() + #expect(config.evictionPolicy == .lru) + } + + @Test func evictionPolicyFIFO() throws { + let policy = EvictionPolicy.fifo + #expect(policy == .fifo) + + let config = CacheConfiguration(evictionPolicy: .fifo) + #expect(config.evictionPolicy == .fifo) + } + + @Test func evictionPolicyARC() throws { + let policy = EvictionPolicy.arc + #expect(policy == .arc) + + let config = CacheConfiguration(evictionPolicy: .arc) + #expect(config.evictionPolicy == .arc) + } + + // MARK: - CompressionConfiguration Tests + + @Test func compressionConfigurationDefault() throws { + let compression = CompressionConfiguration.default + + #expect(compression.algorithm == .zstd) + #expect(compression.level == 3) + #expect(compression.minSize == 1024) + } + + @Test func compressionConfigurationCustomAlgorithms() throws { + let zstdConfig = CompressionConfiguration(algorithm: .zstd, level: 5, minSize: 512) + #expect(zstdConfig.algorithm == .zstd) + #expect(zstdConfig.level == 5) + #expect(zstdConfig.minSize == 512) + + let lz4Config = CompressionConfiguration(algorithm: .lz4, level: 1, minSize: 256) + #expect(lz4Config.algorithm == .lz4) + #expect(lz4Config.level == 1) + #expect(lz4Config.minSize == 256) + + let gzipConfig = CompressionConfiguration(algorithm: .gzip, level: 9, minSize: 2048) + #expect(gzipConfig.algorithm == .gzip) + #expect(gzipConfig.level == 9) + #expect(gzipConfig.minSize == 2048) + + let noneConfig = CompressionConfiguration(algorithm: .none, level: 0, minSize: 0) + #expect(noneConfig.algorithm == .none) + #expect(noneConfig.level == 0) + #expect(noneConfig.minSize == 0) + } + + @Test func compressionConfigurationAlgorithmRawValues() throws { + #expect(CompressionConfiguration.CompressionAlgorithm.zstd.rawValue == "zstd") + #expect(CompressionConfiguration.CompressionAlgorithm.lz4.rawValue == "lz4") + #expect(CompressionConfiguration.CompressionAlgorithm.gzip.rawValue == "gzip") + #expect(CompressionConfiguration.CompressionAlgorithm.none.rawValue == "none") + } + + // MARK: - ConcurrencyConfiguration Tests + + @Test func concurrencyConfigurationDefault() throws { + let concurrency = ConcurrencyConfiguration.default + + #expect(concurrency.maxConcurrentReads == 100) + #expect(concurrency.maxConcurrentWrites == 10) + #expect(concurrency.maxConcurrentEvictions == 2) + } + + @Test func concurrencyConfigurationCustom() throws { + let concurrency = ConcurrencyConfiguration( + maxConcurrentReads: 200, + maxConcurrentWrites: 20, + maxConcurrentEvictions: 5 + ) + + #expect(concurrency.maxConcurrentReads == 200) + #expect(concurrency.maxConcurrentWrites == 20) + #expect(concurrency.maxConcurrentEvictions == 5) + } + + @Test func concurrencyConfigurationMinimalValues() throws { + let concurrency = ConcurrencyConfiguration( + maxConcurrentReads: 1, + maxConcurrentWrites: 1, + maxConcurrentEvictions: 1 + ) + + #expect(concurrency.maxConcurrentReads == 1) + #expect(concurrency.maxConcurrentWrites == 1) + #expect(concurrency.maxConcurrentEvictions == 1) + } + + // MARK: - CacheStatistics Tests + + @Test func cacheStatisticsInitialization() throws { + let operationMetrics = OperationMetrics( + totalOperations: 100, + averageGetDuration: 0.05, + averagePutDuration: 0.1, + p95GetDuration: 0.08, + p95PutDuration: 0.15 + ) + let stats = CacheStatistics( + entryCount: 100, + totalSize: 1024 * 1024, + hitRate: 0.85, + oldestEntryAge: 3600, + mostRecentEntryAge: 60, + evictionPolicy: "lru", + compressionRatio: 0.7, + averageEntrySize: 10240, + operationMetrics: operationMetrics, + errorCount: 0, + lastGCTime: nil, + shardInfo: nil + ) + + #expect(stats.entryCount == 100) + #expect(stats.totalSize == 1024 * 1024) + #expect(abs(stats.hitRate - 0.85) < 0.001) + #expect(stats.oldestEntryAge == 3600) + #expect(stats.mostRecentEntryAge == 60) + } + + @Test func cacheStatisticsEmptyCache() throws { + let operationMetrics = OperationMetrics( + totalOperations: 0, + averageGetDuration: 0.0, + averagePutDuration: 0.0, + p95GetDuration: 0.0, + p95PutDuration: 0.0 + ) + let stats = CacheStatistics( + entryCount: 0, + totalSize: 0, + hitRate: 0.0, + oldestEntryAge: 0, + mostRecentEntryAge: 0, + evictionPolicy: "lru", + compressionRatio: 1.0, + averageEntrySize: 0, + operationMetrics: operationMetrics, + errorCount: 0, + lastGCTime: nil, + shardInfo: nil + ) + + #expect(stats.entryCount == 0) + #expect(stats.totalSize == 0) + #expect(stats.hitRate == 0.0) + #expect(stats.oldestEntryAge == 0) + #expect(stats.mostRecentEntryAge == 0) + } + + // MARK: - Integration Tests + + @Test func cacheConfigurationIntegration() throws { + // Test that all configuration components work together + let tempDir = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString) + + let config = CacheConfiguration( + maxSize: 512 * 1024 * 1024, // 512MB + maxAge: 24 * 60 * 60, // 1 day + compression: CompressionConfiguration(algorithm: .lz4, level: 1, minSize: 512), + indexPath: tempDir.appendingPathComponent("test-cache.db"), + evictionPolicy: .lru, + concurrency: ConcurrencyConfiguration( + maxConcurrentReads: 50, + maxConcurrentWrites: 5, + maxConcurrentEvictions: 1 + ), + verifyIntegrity: true, + sharding: nil, + gcInterval: 900, // 15 minutes + cacheKeyVersion: "test-v1", + defaultTTL: 7200 // 2 hours + ) + + // Verify all settings are preserved + #expect(config.maxSize == 512 * 1024 * 1024) + #expect(config.maxAge == 24 * 60 * 60) + #expect(config.compression.algorithm == .lz4) + #expect(config.compression.level == 1) + #expect(config.compression.minSize == 512) + #expect(config.evictionPolicy == .lru) + #expect(config.concurrency.maxConcurrentReads == 50) + #expect(config.concurrency.maxConcurrentWrites == 5) + #expect(config.concurrency.maxConcurrentEvictions == 1) + #expect(config.verifyIntegrity == true) + #expect(config.sharding == nil) + #expect(config.gcInterval == 900) + #expect(config.cacheKeyVersion == "test-v1") + #expect(config.defaultTTL == 7200) + } + + // MARK: - Edge Cases + + @Test func cacheConfigurationEdgeCases() throws { + // Test with very large values + let largeConfig = CacheConfiguration( + maxSize: UInt64.max, + maxAge: TimeInterval.greatestFiniteMagnitude, + gcInterval: TimeInterval.greatestFiniteMagnitude + ) + + #expect(largeConfig.maxSize == UInt64.max) + #expect(largeConfig.maxAge == TimeInterval.greatestFiniteMagnitude) + #expect(largeConfig.gcInterval == TimeInterval.greatestFiniteMagnitude) + + // Test with minimal values + let minimalConfig = CacheConfiguration( + maxSize: 0, + maxAge: 0, + gcInterval: 0 + ) + + #expect(minimalConfig.maxSize == 0) + #expect(minimalConfig.maxAge == 0) + #expect(minimalConfig.gcInterval == 0) + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildCacheTests/CacheIndexTests.swift b/Tests/NativeBuilderTests/ContainerBuildCacheTests/CacheIndexTests.swift new file mode 100644 index 00000000..07c85ee1 --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildCacheTests/CacheIndexTests.swift @@ -0,0 +1,523 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerizationOCI +import Foundation +import Testing + +@testable import ContainerBuildCache + +struct CacheIndexTests { + + @Test func putAndGet() async throws { + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let cacheIndex = try CacheIndex(path: tempDir) + + // Create test data + let key = "test-key-123" + let descriptor = Descriptor( + mediaType: "application/vnd.test+json", + digest: "sha256:1234567890abcdef", + size: 1024, + urls: nil, + annotations: nil, + platform: nil + ) + let platform = Platform( + arch: "amd64", + os: "linux", + ) + let metadata = CacheMetadata( + operationHash: "op-hash-123", + platform: platform, + ttl: 3600, + tags: ["test": "true"] + ) + + // Test put + try await cacheIndex.put(key: key, descriptor: descriptor, metadata: metadata) + + // Test get + let entry = try await cacheIndex.get(key: key) + #expect(entry != nil) + #expect(entry?.descriptor.digest == descriptor.digest) + #expect(entry?.metadata.operationHash == metadata.operationHash) + + // Test cache.json was created + let cacheJsonPath = tempDir.appendingPathComponent("cache.json") + #expect(FileManager.default.fileExists(atPath: cacheJsonPath.path) == true) + } + + @Test func remove() async throws { + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let cacheIndex = try CacheIndex(path: tempDir) + + // Add entries + let keys = ["key1", "key2", "key3"] + for key in keys { + let descriptor = Descriptor( + mediaType: "application/vnd.test+json", + digest: "sha256:\(key)", + size: 100, + urls: nil, + annotations: nil, + platform: nil + ) + let metadata = CacheMetadata( + operationHash: "hash-\(key)", + platform: Platform(arch: "amd64", os: "linux") + ) + try await cacheIndex.put(key: key, descriptor: descriptor, metadata: metadata) + } + + // Remove some entries + try await cacheIndex.remove(keys: ["key1", "key3"]) + + // Verify + let entry1 = try await cacheIndex.get(key: "key1") + let entry2 = try await cacheIndex.get(key: "key2") + let entry3 = try await cacheIndex.get(key: "key3") + + #expect(entry1 == nil) + #expect(entry2 != nil) + #expect(entry3 == nil) + } + + @Test func statistics() async throws { + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let cacheIndex = try CacheIndex(path: tempDir) + + // Add some entries + for i in 1...5 { + let descriptor = Descriptor( + mediaType: "application/vnd.test+json", + digest: "sha256:entry\(i)", + size: Int64(i * 1000), + urls: nil, + annotations: nil, + platform: nil + ) + let metadata = CacheMetadata( + operationHash: "hash-\(i)", + platform: Platform(arch: "amd64", os: "linux") + ) + try await cacheIndex.put(key: "key\(i)", descriptor: descriptor, metadata: metadata) + } + + // Get some entries to affect hit rate + _ = try await cacheIndex.get(key: "key1") + _ = try await cacheIndex.get(key: "key2") + _ = try await cacheIndex.get(key: "key-missing") // This should be a miss + + let stats = try await cacheIndex.statistics() + + #expect(stats.entryCount == 5) + #expect(stats.totalSize == 15000) // 1000 + 2000 + 3000 + 4000 + 5000 + #expect(stats.averageEntrySize == 3000) + #expect(stats.hitRate > 0.6) // 2 hits out of 3 attempts + } + + @Test func allEntries() async throws { + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let cacheIndex = try CacheIndex(path: tempDir) + + // Add entries + let entries = [ + ("key1", "hash1"), + ("key2", "hash2"), + ("key3", "hash3"), + ] + + for (key, hash) in entries { + let descriptor = Descriptor( + mediaType: "application/vnd.test+json", + digest: "sha256:\(hash)", + size: 100, + urls: nil, + annotations: nil, + platform: nil + ) + let metadata = CacheMetadata( + operationHash: hash, + platform: Platform(arch: "amd64", os: "linux") + ) + try await cacheIndex.put(key: key, descriptor: descriptor, metadata: metadata) + } + + // Get all entries + let allEntries = try await cacheIndex.allEntries() + + #expect(allEntries.count == 3) + #expect(allEntries["key1"] != nil) + #expect(allEntries["key2"] != nil) + #expect(allEntries["key3"] != nil) + } + + @Test func persistence() async throws { + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let cacheIndex = try CacheIndex(path: tempDir) + + // Add an entry + let key = "persistent-key" + let descriptor = Descriptor( + mediaType: "application/vnd.test+json", + digest: "sha256:persistent", + size: 2048, + urls: nil, + annotations: nil, + platform: nil + ) + let metadata = CacheMetadata( + operationHash: "persistent-hash", + platform: Platform(arch: "arm64", os: "linux") + ) + + try await cacheIndex.put(key: key, descriptor: descriptor, metadata: metadata) + + // Create a new cache index with same path + let newCacheIndex = try CacheIndex(path: tempDir) + + // Verify data persisted + let entry = try await newCacheIndex.get(key: key) + #expect(entry != nil) + #expect(entry?.descriptor.digest == descriptor.digest) + #expect(entry?.metadata.platform.os == "linux") + #expect(entry?.metadata.platform.architecture == "arm64") + } + + @Test func accessTimeUpdate() async throws { + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let cacheIndex = try CacheIndex(path: tempDir) + + let key = "access-test" + let descriptor = Descriptor( + mediaType: "application/vnd.test+json", + digest: "sha256:access", + size: 512, + urls: nil, + annotations: nil, + platform: nil + ) + let metadata = CacheMetadata( + operationHash: "access-hash", + platform: Platform(arch: "amd64", os: "linux") + ) + + try await cacheIndex.put(key: key, descriptor: descriptor, metadata: metadata) + + // Get initial access time + let entry1 = try await cacheIndex.get(key: key) + let accessTime1 = entry1?.metadata.accessedAt + + // Wait a bit + try await Task.sleep(nanoseconds: 100_000_000) // 0.1 seconds + + // Access again + let entry2 = try await cacheIndex.get(key: key) + let accessTime2 = entry2?.metadata.accessedAt + + // Access time should be updated + #expect(accessTime1 != nil) + #expect(accessTime2 != nil) + #expect(accessTime2! > accessTime1!) + } + + // MARK: - Additional Tests for Concurrency, Large Datasets, and Error Handling + + @Test func cacheIndexConcurrentAccess() async throws { + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let cacheIndex = try CacheIndex(path: tempDir) + + let entryCount = 50 + + // Test concurrent puts and gets + await withTaskGroup(of: Void.self) { group in + // Concurrent puts + for i in 0.. 0) + } + + @Test func cacheIndexLargeDataSet() async throws { + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let cacheIndex = try CacheIndex(path: tempDir) + + let largeEntryCount = 1000 + + // Add a large number of entries + for i in 0.. 0) + #expect(stats.hitRate > 0.9) // Should have high hit rate + } + + @Test func cacheIndexCorruptedIndexFile() async throws { + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let cacheIndex = try CacheIndex(path: tempDir) + + // Add some entries first + let key = "test-key" + let descriptor = Descriptor( + mediaType: "application/vnd.test+json", + digest: "sha256:test", + size: 1024, + urls: nil, + annotations: nil, + platform: nil + ) + let metadata = CacheMetadata( + operationHash: "test-hash", + platform: Platform(arch: "amd64", os: "linux") + ) + + try await cacheIndex.put(key: key, descriptor: descriptor, metadata: metadata) + + // Verify entry exists + let entry1 = try await cacheIndex.get(key: key) + #expect(entry1 != nil) + + // Corrupt the cache.json file + let cacheJsonPath = tempDir.appendingPathComponent("cache.json") + try "corrupted data".write(to: cacheJsonPath, atomically: true, encoding: .utf8) + + // Create a new cache index - should handle corruption gracefully + let newCacheIndex = try CacheIndex(path: tempDir) + + // Should start with empty state + let entry2 = try await newCacheIndex.get(key: key) + #expect(entry2 == nil) // Entry should be lost due to corruption + + // Should still be functional for new entries + try await newCacheIndex.put(key: "new-key", descriptor: descriptor, metadata: metadata) + let newEntry = try await newCacheIndex.get(key: "new-key") + #expect(newEntry != nil) + } + + @Test func cacheIndexStatisticsAccuracy() async throws { + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let cacheIndex = try CacheIndex(path: tempDir) + + // Test that statistics are accurately maintained + let initialStats = try await cacheIndex.statistics() + #expect(initialStats.entryCount == 0) + #expect(initialStats.totalSize == 0) + #expect(initialStats.hitRate == 0.0) + + // Add entries with known sizes + let entrySizes: [Int64] = [100, 200, 300, 400, 500] + for (i, size) in entrySizes.enumerated() { + let key = "stats-key-\(i)" + let descriptor = Descriptor( + mediaType: "application/vnd.test+json", + digest: "sha256:stats\(i)", + size: size, + urls: nil, + annotations: nil, + platform: nil + ) + let metadata = CacheMetadata( + operationHash: "stats-hash-\(i)", + platform: Platform(arch: "amd64", os: "linux") + ) + + try await cacheIndex.put(key: key, descriptor: descriptor, metadata: metadata) + } + + // Check statistics after puts + let afterPutStats = try await cacheIndex.statistics() + #expect(afterPutStats.entryCount == entrySizes.count) + #expect(afterPutStats.totalSize == UInt64(entrySizes.reduce(0, +))) + let expectedAverage = UInt64(entrySizes.reduce(0, +)) / UInt64(entrySizes.count) + #expect(afterPutStats.averageEntrySize == expectedAverage) + + // Perform some gets (hits and misses) + _ = try await cacheIndex.get(key: "stats-key-0") // Hit + _ = try await cacheIndex.get(key: "stats-key-1") // Hit + _ = try await cacheIndex.get(key: "stats-key-2") // Hit + _ = try await cacheIndex.get(key: "nonexistent-1") // Miss + _ = try await cacheIndex.get(key: "nonexistent-2") // Miss + + // Check final statistics + let finalStats = try await cacheIndex.statistics() + #expect(finalStats.entryCount == entrySizes.count) + #expect(finalStats.totalSize == UInt64(entrySizes.reduce(0, +))) + // Hit rate should be 3 hits out of 5 total operations (3 hits + 2 misses) + #expect(abs(finalStats.hitRate - 3.0 / 5.0) < 0.001) + } + + @Test func cacheIndexTTLExpiration() async throws { + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + let cacheIndex = try CacheIndex(path: tempDir) + + // Test TTL-based expiration detection + let key = "ttl-test" + let descriptor = Descriptor( + mediaType: "application/vnd.test+json", + digest: "sha256:ttl", + size: 1024, + urls: nil, + annotations: nil, + platform: nil + ) + + // Create metadata with short TTL + let now = Date() + let metadata = CacheMetadata( + createdAt: now, + accessedAt: now, + operationHash: "ttl-hash", + platform: Platform(arch: "amd64", os: "linux"), + ttl: 2.0 // 2 seconds to account for test execution time + ) + + try await cacheIndex.put(key: key, descriptor: descriptor, metadata: metadata) + + // Immediately check - should not be expired + let entry1 = try await cacheIndex.get(key: key) + #expect(entry1 != nil) + #expect(entry1!.metadata.isExpired == false) // Entry should not be expired immediately after creation + + // Wait for TTL to expire + try await Task.sleep(nanoseconds: 2_100_000_000) // 2.1 seconds + + // Check expiration status + let entry2 = try await cacheIndex.get(key: key) + #expect(entry2 != nil) // Entry still exists in index + #expect(entry2!.metadata.isExpired == true) // But is marked as expired + } + + @Test func cacheIndexEmptyDirectory() async throws { + // Test creating cache index in empty directory + let emptyDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: emptyDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: emptyDir) } + + let emptyCacheIndex = try CacheIndex(path: emptyDir) + + // Should start with empty statistics + let stats = try await emptyCacheIndex.statistics() + #expect(stats.entryCount == 0) + #expect(stats.totalSize == 0) + + // Should be functional + let key = "empty-test" + let descriptor = Descriptor( + mediaType: "application/vnd.test+json", + digest: "sha256:empty", + size: 512, + urls: nil, + annotations: nil, + platform: nil + ) + let metadata = CacheMetadata( + operationHash: "empty-hash", + platform: Platform(arch: "amd64", os: "linux") + ) + + try await emptyCacheIndex.put(key: key, descriptor: descriptor, metadata: metadata) + let entry = try await emptyCacheIndex.get(key: key) + #expect(entry != nil) + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildCacheTests/CacheIntegrationTests.swift b/Tests/NativeBuilderTests/ContainerBuildCacheTests/CacheIntegrationTests.swift new file mode 100644 index 00000000..80b72a51 --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildCacheTests/CacheIntegrationTests.swift @@ -0,0 +1,446 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildSnapshotter +import Foundation +import Testing + +@testable import ContainerBuildCache + +struct CacheIntegrationTests { + + // MARK: - Complete Workflow Tests + + @Test func cacheIntegrationCompleteWorkflow() async throws { + try await withCacheTestEnvironment { (testEnv: CacheTestEnvironment) in + let tempDir = testEnv.tempDir + let mockContentStore = MockContentStore(baseDir: testEnv.tempDir) + let config = TestDataFactory.createCacheConfiguration( + indexPath: tempDir.appendingPathComponent("workflow-cache") + ) + + // Initialize cache + let cache = try await ContentAddressableCache( + contentStore: mockContentStore, + configuration: config + ) + + // Create test data + let operation = TestDataFactory.createOperation(kind: "build", content: "compile-app") + let key = TestDataFactory.createCacheKey(operationContent: "build-operation") + let result = TestDataFactory.createCachedResult( + snapshotContent: "compiled-app", + environmentChanges: [ + "PATH": .literal("/usr/local/bin:/usr/bin"), + "CC": .literal("clang"), + ], + metadataChanges: [ + "build.time": "2024-01-01T12:00:00Z", + "build.version": "1.0.0", + ] + ) + + // Step 1: Cache miss + let initialResult = await cache.get(key, for: operation) + #expect(initialResult == nil, "Initial cache lookup should miss") + + // Step 2: Store result + await cache.put(result, key: key, for: operation) + + // Step 3: Cache hit + let cachedResult = await cache.get(key, for: operation) + #expect(cachedResult != nil, "Cache lookup should hit after storing") + #expect(cachedResult?.snapshot.digest == result.snapshot.digest) + #expect(cachedResult?.environmentChanges.count == result.environmentChanges.count) + #expect(cachedResult?.metadataChanges == result.metadataChanges) + + // Step 4: Verify statistics + let stats = await cache.statistics() + #expect(stats.entryCount == 1) + #expect(stats.totalSize > 0) + #expect(stats.hitRate > 0) + } + } + + @Test func cacheIntegrationMultipleOperations() async throws { + try await withCacheTestEnvironment { (testEnv: CacheTestEnvironment) in + let tempDir = testEnv.tempDir + let mockContentStore = MockContentStore(baseDir: testEnv.tempDir) + let config = TestDataFactory.createCacheConfiguration( + indexPath: tempDir.appendingPathComponent("multi-op-cache") + ) + + let cache = try await ContentAddressableCache( + contentStore: mockContentStore, + configuration: config + ) + + // Simulate a multi-stage build process + let operations = [ + ("download", "Download dependencies"), + ("compile", "Compile source code"), + ("test", "Run unit tests"), + ("package", "Create distribution package"), + ] + + var cachedResults: [CachedResult] = [] + + // Store results for each operation + for (i, (opType, opDescription)) in operations.enumerated() { + let operation = TestDataFactory.createOperation(kind: opType, content: opDescription) + let key = TestDataFactory.createCacheKey( + operationContent: "\(opType)-\(i)", + inputContents: i > 0 ? ["previous-stage-\(i-1)"] : [] + ) + let result = TestDataFactory.createCachedResult( + snapshotContent: "\(opType)-result-\(i)", + environmentChanges: ["STAGE": .literal(opType)], + metadataChanges: ["stage.name": opType, "stage.index": "\(i)"] + ) + + await cache.put(result, key: key, for: operation) + cachedResults.append(result) + } + + // Verify all operations are cached + for (i, (opType, opDescription)) in operations.enumerated() { + let operation = TestDataFactory.createOperation(kind: opType, content: opDescription) + let key = TestDataFactory.createCacheKey( + operationContent: "\(opType)-\(i)", + inputContents: i > 0 ? ["previous-stage-\(i-1)"] : [] + ) + + let cachedResult = await cache.get(key, for: operation) + #expect(cachedResult != nil, "Operation \(opType) should be cached") + #expect(cachedResult?.snapshot.digest == cachedResults[i].snapshot.digest) + } + + // Verify final statistics + let stats = await cache.statistics() + #expect(stats.entryCount == operations.count) + #expect(stats.totalSize > 0) + } + } + + @Test func cacheIntegrationPersistenceAcrossRestarts() async throws { + try await withCacheTestEnvironment { (testEnv: CacheTestEnvironment) in + let mockContentStore = MockContentStore(baseDir: testEnv.tempDir) + let indexPath = testEnv.tempDir.appendingPathComponent("persistent-cache") + let config = TestDataFactory.createCacheConfiguration(indexPath: indexPath) + + let operation = TestDataFactory.createOperation() + let key = TestDataFactory.createCacheKey() + let result = TestDataFactory.createCachedResult() + + // First cache instance + do { + let cache1 = try await ContentAddressableCache( + contentStore: mockContentStore, + configuration: config + ) + + await cache1.put(result, key: key, for: operation) + + let cachedResult1 = await cache1.get(key, for: operation) + #expect(cachedResult1 != nil) + } + + // Second cache instance (simulating restart) + do { + let cache2 = try await ContentAddressableCache( + contentStore: mockContentStore, + configuration: config + ) + + // Should still find the cached result + let cachedResult2 = await cache2.get(key, for: operation) + #expect(cachedResult2 != nil, "Cache should persist across restarts") + #expect(cachedResult2?.snapshot.digest == result.snapshot.digest) + } + } + } + + @Test func cacheIntegrationLargeDataSets() async throws { + try await withCacheTestEnvironment { (testEnv: CacheTestEnvironment) in + let mockContentStore = MockContentStore(baseDir: testEnv.tempDir) + let config = TestDataFactory.createCacheConfiguration( + maxSize: 10 * 1024 * 1024, // 10MB + indexPath: testEnv.tempDir.appendingPathComponent("large-data-cache") + ) + + let cache = try await ContentAddressableCache( + contentStore: mockContentStore, + configuration: config + ) + + let operation = TestDataFactory.createOperation() + let entryCount = 100 + + // Store many entries + for i in 0.. 0, "Should have at least some cached entries") + + let stats = await cache.statistics() + #expect(stats.totalSize <= config.maxSize * 2) // Allow some overhead + } + } + + @Test func cacheIntegrationErrorRecovery() async throws { + try await withCacheTestEnvironment { (testEnv: CacheTestEnvironment) in + let mockContentStore = MockContentStore(baseDir: testEnv.tempDir) + let config = TestDataFactory.createCacheConfiguration( + indexPath: testEnv.tempDir.appendingPathComponent("error-recovery-cache") + ) + + let cache = try await ContentAddressableCache( + contentStore: mockContentStore, + configuration: config + ) + + let operation = TestDataFactory.createOperation() + + // Store some valid entries + for i in 0..<5 { + let key = TestDataFactory.createCacheKey(operationContent: "valid-entry-\(i)") + let result = TestDataFactory.createCachedResult(snapshotContent: "valid-content-\(i)") + await cache.put(result, key: key, for: operation) + } + + // Verify cache is working + let stats1 = await cache.statistics() + #expect(stats1.entryCount == 5) + + // Simulate error condition by clearing content store but keeping index + await mockContentStore.clear() + + // Cache should handle missing content gracefully + let key = TestDataFactory.createCacheKey(operationContent: "valid-entry-0") + let _ = await cache.get(key, for: operation) + // Result should be nil due to missing content, but cache shouldn't crash + + // Cache should still be functional for new entries + let newKey = TestDataFactory.createCacheKey(operationContent: "new-entry") + let newResult = TestDataFactory.createCachedResult(snapshotContent: "new-content") + await cache.put(newResult, key: newKey, for: operation) + + let cachedNewResult = await cache.get(newKey, for: operation) + #expect(cachedNewResult != nil, "Cache should recover and work for new entries") + } + } + + // MARK: - Cross-Cache Type Integration + + @Test func cacheIntegrationMemoryCacheComparison() async throws { + try await withCacheTestEnvironment { (testEnv: CacheTestEnvironment) in + let memoryCache = MemoryBuildCache() + + let mockContentStore = MockContentStore(baseDir: testEnv.tempDir) + let config = TestDataFactory.createCacheConfiguration( + indexPath: testEnv.tempDir.appendingPathComponent("comparison-cache") + ) + let persistentCache = try await ContentAddressableCache( + contentStore: mockContentStore, + configuration: config + ) + + let operation = TestDataFactory.createOperation() + let key = TestDataFactory.createCacheKey() + let result = TestDataFactory.createCachedResult() + + // Store in both caches + await memoryCache.put(result, key: key, for: operation) + await persistentCache.put(result, key: key, for: operation) + + // Retrieve from both caches + let memoryResult = await memoryCache.get(key, for: operation) + let persistentResult = await persistentCache.get(key, for: operation) + + // Both should return the same logical result + #expect(memoryResult != nil) + #expect(persistentResult != nil) + #expect(memoryResult?.snapshot.digest == persistentResult?.snapshot.digest) + #expect(memoryResult?.environmentChanges.count == persistentResult?.environmentChanges.count) + #expect(memoryResult?.metadataChanges == persistentResult?.metadataChanges) + + // Compare statistics + let memoryStats = await memoryCache.statistics() + let persistentStats = await persistentCache.statistics() + + #expect(memoryStats.entryCount == persistentStats.entryCount) + // Note: totalSize might differ due to different storage mechanisms + } + } + + // MARK: - Performance Integration Tests + + @Test func cacheIntegrationPerformanceUnderLoad() async throws { + try await withCacheTestEnvironment { (testEnv: CacheTestEnvironment) in + let mockContentStore = MockContentStore(baseDir: testEnv.tempDir) + let config = TestDataFactory.createCacheConfiguration( + maxSize: 50 * 1024 * 1024, // 50MB + indexPath: testEnv.tempDir.appendingPathComponent("performance-cache") + ) + + let cache = try await ContentAddressableCache( + contentStore: mockContentStore, + configuration: config + ) + + let operation = TestDataFactory.createOperation() + let operationCount = 50 + + // Measure performance under concurrent load + let (_, duration) = await PerformanceMeasurement.measure { + // First, perform concurrent writes + await withTaskGroup(of: Void.self) { group in + for i in 0.. 0) + #expect(stats.hitRate > 0) + } + } + + // MARK: - Real-World Scenario Tests + + @Test func cacheIntegrationBuildPipelineScenario() async throws { + try await withCacheTestEnvironment { (testEnv: CacheTestEnvironment) in + let mockContentStore = MockContentStore(baseDir: testEnv.tempDir) + let config = TestDataFactory.createCacheConfiguration( + indexPath: testEnv.tempDir.appendingPathComponent("pipeline-cache") + ) + + let cache = try await ContentAddressableCache( + contentStore: mockContentStore, + configuration: config + ) + + // Simulate a typical build pipeline + let pipeline = [ + ("fetch-sources", ["Dockerfile", "src/"]), + ("install-deps", ["package.json", "yarn.lock"]), + ("compile", ["src/main.ts", "src/utils.ts"]), + ("test", ["test/unit.test.ts"]), + ("build-image", ["dist/", "Dockerfile"]), + ] + + var previousOutputs: [String] = [] + + for (stageName, inputs) in pipeline { + let operation = TestDataFactory.createOperation(kind: stageName, content: "Pipeline stage: \(stageName)") + let key = TestDataFactory.createCacheKey( + operationContent: stageName, + inputContents: inputs + previousOutputs + ) + + // Check cache first + let cachedResult = await cache.get(key, for: operation) + + if cachedResult != nil { + // Cache hit - use cached result + previousOutputs.append("cached-\(stageName)") + print("Cache hit for stage: \(stageName)") + } else { + // Cache miss - simulate work and store result + let result = TestDataFactory.createCachedResult( + snapshotContent: "output-of-\(stageName)", + environmentChanges: ["STAGE": .literal(stageName)], + metadataChanges: [ + "stage": stageName, + "timestamp": ISO8601DateFormatter().string(from: Date()), + ] + ) + + await cache.put(result, key: key, for: operation) + previousOutputs.append("fresh-\(stageName)") + print("Cache miss for stage: \(stageName)") + } + } + + // Verify all stages were processed + #expect(previousOutputs.count == pipeline.count) + + // Run pipeline again - should have more cache hits + var secondRunOutputs: [String] = [] + previousOutputs = [] // Reset for second run + + for (stageName, inputs) in pipeline { + let operation = TestDataFactory.createOperation(kind: stageName, content: "Pipeline stage: \(stageName)") + let key = TestDataFactory.createCacheKey( + operationContent: stageName, + inputContents: inputs + previousOutputs + ) + + let cachedResult = await cache.get(key, for: operation) + if cachedResult != nil { + secondRunOutputs.append("cached-\(stageName)") + previousOutputs.append("cached-\(stageName)") + } else { + secondRunOutputs.append("fresh-\(stageName)") + previousOutputs.append("fresh-\(stageName)") + } + } + + // Second run should have at least some cache hits + let cacheHits = secondRunOutputs.filter { $0.hasPrefix("cached-") }.count + #expect(cacheHits > 0, "Second pipeline run should have cache hits") + + let finalStats = await cache.statistics() + #expect(finalStats.hitRate > 0, "Overall hit rate should be positive") + } + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildCacheTests/CacheTestHelpers.swift b/Tests/NativeBuilderTests/ContainerBuildCacheTests/CacheTestHelpers.swift new file mode 100644 index 00000000..b4128588 --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildCacheTests/CacheTestHelpers.swift @@ -0,0 +1,555 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildSnapshotter +import ContainerizationOCI +import Crypto +import Foundation +import Testing + +@testable import ContainerBuildCache + +// MARK: - Missing Type Definitions for Testing + +/// ContentWriter mock for testing +public struct ContentWriter { + public let ingestDir: URL + + public init(for ingestDir: URL) throws { + self.ingestDir = ingestDir + } + + public func write(_ data: Data) throws -> (Int64, SHA256.Digest) { + let digest = SHA256.hash(data: data) + // Use the digest string without the sha256: prefix for the filename + let digestString = digest.map { String(format: "%02x", $0) }.joined() + let filePath = ingestDir.appendingPathComponent(digestString) + try data.write(to: filePath) + return (Int64(data.count), digest) + } + + public func create(from manifest: CacheManifest) throws -> (Int64, SHA256.Digest) { + let data = try JSONEncoder().encode(manifest) + return try write(data) + } +} + +extension Data { + var sha256: String { + let digest = (try? ContainerBuildIR.Digest.compute(self, using: .sha256)) ?? (try! ContainerBuildIR.Digest(algorithm: .sha256, bytes: Data(count: 32))) + return digest.stringValue.replacingOccurrences(of: "sha256:", with: "") + } +} + +// MARK: - Test Environment + +/// Test environment with common setup and utilities for cache tests +public struct CacheTestEnvironment { + public let tempDir: URL + + public init() throws { + self.tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + } + + public func cleanup() { + try? FileManager.default.removeItem(at: tempDir) + } +} + +/// Convenience function to create and manage test environment +public func withCacheTestEnvironment( + _ operation: (CacheTestEnvironment) async throws -> T +) async throws -> T { + let environment = try CacheTestEnvironment() + defer { environment.cleanup() } + return try await operation(environment) +} + +// MARK: - Compatibility Layer + +/// Compatibility base class for existing XCTest-based tests during migration +/// This maintains the same interface as the original CacheTestCase for gradual migration +open class CacheTestCase { + public var tempDir: URL! + + public init() {} + + open func setUp() async throws { + tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + } + + open func tearDown() async throws { + if let tempDir = tempDir { + try? FileManager.default.removeItem(at: tempDir) + } + } +} + +// MARK: - ContentStore Protocol +// Using the real ContentStore from ContainerizationOCI + +// MARK: - Mock Content + +/// Mock Content implementation for testing +public struct MockContent: Content { + public let path: URL + private let _data: Data + + public init(data: Data) { + self._data = data + self.path = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + } + + public func digest() throws -> SHA256.Digest { + SHA256.hash(data: _data) + } + + public func size() throws -> UInt64 { + UInt64(_data.count) + } + + public func data() throws -> Data { + _data + } + + public func data(offset: UInt64, length: Int) throws -> Data? { + let start = Int(offset) + let end = min(start + length, _data.count) + guard start < _data.count else { return nil } + return _data.subdata(in: start..() throws -> T { + let decoder = JSONDecoder() + return try decoder.decode(T.self, from: _data) + } +} + +// MARK: - Mock ContentStore + +/// Mock ContentStore for testing cache implementations +public actor MockContentStore: ContentStore { + private var storage: [String: Data] = [:] + private var manifests: [String: CacheManifest] = [:] + private var sessions: [String: URL] = [:] + private var nextSessionId = 0 + private let baseDir: URL + + public init(baseDir: URL? = nil) { + self.baseDir = baseDir ?? FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try? FileManager.default.createDirectory(at: self.baseDir, withIntermediateDirectories: true) + } + + // MARK: - ContentStore Interface + + // MARK: - ContentStore Protocol Implementation + + public func get(digest: String) async throws -> (any Content)? { + guard let data = storage[digest] else { + return nil + } + // Return a mock Content object + return MockContent(data: data) + } + + public func get(digest: String) async throws -> T? { + guard let data = storage[digest] else { + return nil + } + let decoder = JSONDecoder() + return try decoder.decode(T.self, from: data) + } + + public func put(_ object: T, digest: String) async throws { + let encoder = JSONEncoder() + let data = try encoder.encode(object) + storage[digest] = data + } + + @discardableResult + public func delete(digests: [String]) async throws -> ([String], UInt64) { + var deletedDigests: [String] = [] + var totalSize: UInt64 = 0 + + for digest in digests { + if let data = storage.removeValue(forKey: digest) { + deletedDigests.append(digest) + totalSize += UInt64(data.count) + } + manifests.removeValue(forKey: digest) + } + + return (deletedDigests, totalSize) + } + + @discardableResult + public func delete(keeping: [String]) async throws -> ([String], UInt64) { + let keepSet = Set(keeping) + var deletedDigests: [String] = [] + var totalSize: UInt64 = 0 + + for (digest, data) in storage { + if !keepSet.contains(digest) { + storage.removeValue(forKey: digest) + manifests.removeValue(forKey: digest) + deletedDigests.append(digest) + totalSize += UInt64(data.count) + } + } + + return (deletedDigests, totalSize) + } + + @discardableResult + public func ingest(_ body: @Sendable @escaping (URL) async throws -> Void) async throws -> [String] { + let tempDir = baseDir.appendingPathComponent("ingest-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + + try await body(tempDir) + + // Mock implementation - return empty array + return [] + } + + public func newIngestSession() async throws -> (id: String, ingestDir: URL) { + nextSessionId += 1 + let sessionId = "session-\(nextSessionId)" + let sessionDir = baseDir.appendingPathComponent(sessionId) + try FileManager.default.createDirectory(at: sessionDir, withIntermediateDirectories: true) + sessions[sessionId] = sessionDir + return (id: sessionId, ingestDir: sessionDir) + } + + @discardableResult + public func completeIngestSession(_ sessionId: String) async throws -> [String] { + guard let sessionDir = sessions[sessionId] else { + throw MockContentStoreError.sessionNotFound(sessionId) + } + + // Read all files from the session directory and store them + var digests: [String] = [] + + if FileManager.default.fileExists(atPath: sessionDir.path) { + let files = try FileManager.default.contentsOfDirectory(at: sessionDir, includingPropertiesForKeys: nil) + for file in files { + let data = try Data(contentsOf: file) + let digestHash = file.lastPathComponent + // Ensure the digest has the sha256: prefix for storage + let digest = digestHash.hasPrefix("sha256:") ? digestHash : "sha256:\(digestHash)" + storage[digest] = data + digests.append(digest) + } + } + + // Clean up session + sessions.removeValue(forKey: sessionId) + try? FileManager.default.removeItem(at: sessionDir) + + return digests + } + + public func cancelIngestSession(_ sessionId: String) async throws { + guard let sessionDir = sessions[sessionId] else { + throw MockContentStoreError.sessionNotFound(sessionId) + } + + sessions.removeValue(forKey: sessionId) + try? FileManager.default.removeItem(at: sessionDir) + } + + // MARK: - Test Utilities + + public func hasContent(digest: String) async -> Bool { + storage[digest] != nil + } + + public func clear() async { + storage.removeAll() + manifests.removeAll() + for (_, sessionDir) in sessions { + try? FileManager.default.removeItem(at: sessionDir) + } + sessions.removeAll() + } + + public func contentCount() async -> Int { + storage.count + } +} + +// MARK: - Mock ContentStore Errors + +public enum MockContentStoreError: LocalizedError { + case notFound(String) + case sessionNotFound(String) + case encodingFailed(Error) + + public var errorDescription: String? { + switch self { + case .notFound(let digest): + return "Content not found: \(digest)" + case .sessionNotFound(let sessionId): + return "Session not found: \(sessionId)" + case .encodingFailed(let error): + return "Encoding failed: \(error.localizedDescription)" + } + } +} + +// MARK: - Test Data Factory + +/// Factory for creating test data objects +public enum TestDataFactory { + + public static func createDigest(from string: String = "test-content") -> ContainerBuildIR.Digest { + let data = string.data(using: .utf8)! + return (try? ContainerBuildIR.Digest.compute(data, using: .sha256)) ?? (try! ContainerBuildIR.Digest(algorithm: .sha256, bytes: Data(count: 32))) + } + + public static func createSnapshot( + id: UUID = UUID(), + content: String = "test-snapshot", + size: Int64 = 1024, + parent: UUID? = nil + ) -> Snapshot { + let digest = createDigest(from: content) + return Snapshot( + id: id, + digest: digest, + size: size, + parent: parent + ) + } + + public static func createCacheKey( + operation: ContainerBuildIR.Operation? = nil, + operationContent: String = "test-operation", + inputContents: [String] = ["input1", "input2"], + platform: Platform = .linuxAMD64 + ) -> ContainerBuildCache.CacheKey { + let operationDigest: ContainerBuildIR.Digest + if let operation = operation { + operationDigest = (try? operation.contentDigest()) ?? createDigest(from: operationContent) + } else { + operationDigest = createDigest(from: operationContent) + } + let inputDigests = inputContents.map { createDigest(from: $0) } + + return ContainerBuildCache.CacheKey( + operationDigest: operationDigest, + inputDigests: inputDigests, + platform: platform + ) + } + + public static func createCachedResult( + snapshotContent: String = "test-result", + environmentChanges: [String: EnvironmentValue] = ["PATH": .literal("/usr/bin")], + metadataChanges: [String: String] = ["build.time": "2024-01-01T12:00:00Z"] + ) -> CachedResult { + let snapshot = createSnapshot(content: snapshotContent) + return CachedResult( + snapshot: snapshot, + environmentChanges: environmentChanges, + metadataChanges: metadataChanges + ) + } + + public static func createOperation( + kind: String = "test", + content: String = "test-operation" + ) -> MockOperation { + MockOperation(kind: kind, content: content) + } + + public static func createCacheConfiguration( + maxSize: UInt64 = 1024 * 1024, // 1MB for tests + maxAge: TimeInterval = 3600, // 1 hour for tests + indexPath: URL? = nil + ) -> CacheConfiguration { + let testIndexPath = + indexPath + ?? FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString) + .appendingPathComponent("test-cache.db") + + return CacheConfiguration( + maxSize: maxSize, + maxAge: maxAge, + indexPath: testIndexPath, + evictionPolicy: .lru, + verifyIntegrity: false, // Disable for faster tests + gcInterval: 60 // Short interval for tests + ) + } + + public static func createCacheMetadata( + operationHash: String = "test-hash", + platform: Platform = .linuxAMD64, + ttl: TimeInterval? = nil, + tags: [String: String] = [:] + ) -> CacheMetadata { + CacheMetadata( + operationHash: operationHash, + platform: platform, + ttl: ttl, + tags: tags + ) + } + + public static func createDescriptor( + mediaType: String = "application/vnd.test+json", + digest: String = "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + size: Int64 = 1024 + ) -> Descriptor { + Descriptor( + mediaType: mediaType, + digest: digest, + size: size, + urls: nil, + annotations: nil, + platform: nil + ) + } +} + +// MARK: - Mock Operation + +public struct MockOperation: ContainerBuildIR.Operation { + public let kind: String + public let content: String + public let metadata: OperationMetadata + + public static let operationKind = OperationKind(rawValue: "mock") + public var operationKind: OperationKind { + OperationKind(rawValue: kind) + } + + public init(kind: String = "mock", content: String = "test", metadata: OperationMetadata = OperationMetadata()) { + self.kind = kind + self.content = content + self.metadata = metadata + } + + public func accept(_ visitor: V) throws -> V.Result { + try visitor.visitUnknown(self) + } +} + +extension MockOperation: Hashable { + public func hash(into hasher: inout Hasher) { + hasher.combine(kind) + hasher.combine(content) + } + + public static func == (lhs: MockOperation, rhs: MockOperation) -> Bool { + lhs.kind == rhs.kind && lhs.content == rhs.content + } +} + +// MARK: - Performance Measurement + +/// Utility for measuring test performance +public struct PerformanceMeasurement { + public static func measure( + _ operation: () async throws -> T, + file: StaticString = #file, + line: UInt = #line + ) async rethrows -> (result: T, duration: TimeInterval) { + let startTime = Date() + let result = try await operation() + let endTime = Date() + let duration = endTime.timeIntervalSince(startTime) + + print("Performance measurement at \(file):\(line): \(duration)s") + return (result, duration) + } + + public static func measureAndAssert( + _ operation: () async throws -> T, + maxDuration: TimeInterval, + file: StaticString = #file, + line: UInt = #line + ) async rethrows -> T { + let (result, duration) = try await measure(operation, file: file, line: line) + if duration >= maxDuration { + Issue.record( + "Operation took too long: \(duration)s >= \(maxDuration)s", + sourceLocation: SourceLocation(fileID: file.description, filePath: file.description, line: Int(line), column: 1)) + } + return result + } +} + +// MARK: - Async Test Utilities + +/// Utilities for async testing +public enum AsyncTestUtilities { + + /// Wait for a condition to become true with timeout + public static func waitFor( + condition: @escaping () async -> Bool, + timeout: TimeInterval = 5.0, + interval: TimeInterval = 0.1 + ) async throws { + let deadline = Date().addingTimeInterval(timeout) + + while Date() < deadline { + if await condition() { + return + } + try await Task.sleep(nanoseconds: UInt64(interval * 1_000_000_000)) + } + + throw AsyncTestError.timeout + } + + /// Run multiple async operations concurrently and collect results + public static func runConcurrently( + count: Int, + operation: @escaping @Sendable (Int) async throws -> T + ) async throws -> [T] { + try await withThrowingTaskGroup(of: T.self) { group in + for i in 0.. 0) + } + } + + // MARK: - Eviction Tests + + @Test func contentAddressableCacheEvictionBySize() async throws { + try await withCacheTestEnvironment { environment in + let mockContentStore = MockContentStore(baseDir: environment.tempDir) + + // Create a small cache + let smallConfig = TestDataFactory.createCacheConfiguration( + maxSize: 2048, // Very small cache + indexPath: environment.tempDir.appendingPathComponent("small-cache-index") + ) + + let smallCache = try await ContentAddressableCache( + contentStore: mockContentStore, + configuration: smallConfig + ) + + // Fill the cache beyond capacity + let operation = TestDataFactory.createOperation() + for i in 0..<10 { + let key = TestDataFactory.createCacheKey(operationContent: "operation-\(i)") + let result = TestDataFactory.createCachedResult(snapshotContent: "large-content-\(i)") + await smallCache.put(result, key: key, for: operation) + } + + // Give eviction time to run + try await Task.sleep(nanoseconds: 100_000_000 * 5) // 0.1 seconds + + let stats = await smallCache.statistics() + #expect(stats.totalSize < 2048 * 2) // Should have evicted some entries + } + } + + @Test func contentAddressableCacheEvictionByAge() async throws { + try await withCacheTestEnvironment { environment in + let mockContentStore = MockContentStore(baseDir: environment.tempDir) + let configuration = TestDataFactory.createCacheConfiguration( + maxSize: 1024 * 1024, // 1MB for tests + maxAge: 3600, // 1 hour for tests + indexPath: environment.tempDir.appendingPathComponent("cache-index") + ) + + let cache = try await ContentAddressableCache( + contentStore: mockContentStore, + configuration: configuration + ) + + // This test would require manipulating time or using a mock clock + // For now, we'll test the TTL-based eviction logic + + let key = TestDataFactory.createCacheKey() + let result = TestDataFactory.createCachedResult() + let operation = TestDataFactory.createOperation() + + await cache.put(result, key: key, for: operation) + + // Verify it's there + let cachedResult = await cache.get(key, for: operation) + #expect(cachedResult != nil) + + // In a real test, we'd advance time and check eviction + // For now, just verify the entry exists + let stats = await cache.statistics() + #expect(stats.entryCount == 1) + } + } + + @Test func contentAddressableCacheEvictionByTTL() async throws { + try await withCacheTestEnvironment { environment in + let mockContentStore = MockContentStore(baseDir: environment.tempDir) + + // Create configuration with short TTL + let shortTTLConfig = TestDataFactory.createCacheConfiguration( + maxSize: 1024 * 1024, + maxAge: 1, // 1 second TTL + indexPath: environment.tempDir.appendingPathComponent("ttl-cache-index") + ) + + let ttlCache = try await ContentAddressableCache( + contentStore: mockContentStore, + configuration: shortTTLConfig + ) + + let key = TestDataFactory.createCacheKey() + let result = TestDataFactory.createCachedResult() + let operation = TestDataFactory.createOperation() + + await ttlCache.put(result, key: key, for: operation) + + // Verify it's there initially + let initialResult = await ttlCache.get(key, for: operation) + #expect(initialResult != nil) + + // Wait for TTL to expire + try await Task.sleep(nanoseconds: 1_500_000_000) // 1.5 seconds + + // Entry should be evicted (this depends on the implementation running periodic cleanup) + // Note: This test might be flaky depending on the eviction implementation + } + } + + // MARK: - Concurrency Tests + + @Test func contentAddressableCacheConcurrentOperations() async throws { + try await withCacheTestEnvironment { environment in + let mockContentStore = MockContentStore(baseDir: environment.tempDir) + let configuration = TestDataFactory.createCacheConfiguration( + maxSize: 1024 * 1024, // 1MB for tests + maxAge: 3600, // 1 hour for tests + indexPath: environment.tempDir.appendingPathComponent("cache-index") + ) + + let cache = try await ContentAddressableCache( + contentStore: mockContentStore, + configuration: configuration + ) + + let operation = TestDataFactory.createOperation() + + // Test concurrent puts and gets + await withTaskGroup(of: Void.self) { group in + // Concurrent puts + for i in 0..<5 { + group.addTask { + let key = TestDataFactory.createCacheKey(operationContent: "concurrent-op-\(i)") + let result = TestDataFactory.createCachedResult(snapshotContent: "content-\(i)") + await cache.put(result, key: key, for: operation) + } + } + + // Concurrent gets + for i in 0..<5 { + group.addTask { + let key = TestDataFactory.createCacheKey(operationContent: "concurrent-op-\(i)") + _ = await cache.get(key, for: operation) + } + } + } + + let stats = await cache.statistics() + #expect(stats.entryCount > 0) + } + } + + // MARK: - Error Handling Tests + + @Test func contentAddressableCacheCorruptedData() async throws { + try await withCacheTestEnvironment { environment in + let mockContentStore = MockContentStore(baseDir: environment.tempDir) + let configuration = TestDataFactory.createCacheConfiguration( + maxSize: 1024 * 1024, // 1MB for tests + maxAge: 3600, // 1 hour for tests + indexPath: environment.tempDir.appendingPathComponent("cache-index") + ) + + let cache = try await ContentAddressableCache( + contentStore: mockContentStore, + configuration: configuration + ) + + // This test would require injecting corrupted data into the content store + // For now, we'll test basic error resilience + + let key = TestDataFactory.createCacheKey() + let operation = TestDataFactory.createOperation() + + // Try to get from empty cache (should handle gracefully) + let result = await cache.get(key, for: operation) + #expect(result == nil) + + // Cache should still be functional + let stats = await cache.statistics() + #expect(stats.entryCount == 0) + } + } + + @Test func contentAddressableCacheMissingContentStore() async throws { + try await withCacheTestEnvironment { environment in + let mockContentStore = MockContentStore(baseDir: environment.tempDir) + let configuration = TestDataFactory.createCacheConfiguration( + maxSize: 1024 * 1024, // 1MB for tests + maxAge: 3600, // 1 hour for tests + indexPath: environment.tempDir.appendingPathComponent("cache-index") + ) + + let cache = try await ContentAddressableCache( + contentStore: mockContentStore, + configuration: configuration + ) + + // Test behavior when content store operations fail + // This would require a mock that can simulate failures + + let key = TestDataFactory.createCacheKey() + let result = TestDataFactory.createCachedResult() + let operation = TestDataFactory.createOperation() + + // Put should not crash even if content store has issues + await cache.put(result, key: key, for: operation) + + // Get should handle missing content gracefully + let _ = await cache.get(key, for: operation) + // Result depends on whether the put succeeded despite content store issues + } + } + + // MARK: - Performance Tests + + @Test func contentAddressableCachePerformance() async throws { + try await withCacheTestEnvironment { environment in + let mockContentStore = MockContentStore(baseDir: environment.tempDir) + let configuration = TestDataFactory.createCacheConfiguration( + maxSize: 1024 * 1024, // 1MB for tests + maxAge: 3600, // 1 hour for tests + indexPath: environment.tempDir.appendingPathComponent("cache-index") + ) + + let cache = try await ContentAddressableCache( + contentStore: mockContentStore, + configuration: configuration + ) + + let operation = TestDataFactory.createOperation() + + let (_, duration) = await PerformanceMeasurement.measure { + // Perform multiple cache operations + for i in 0..<100 { + let key = TestDataFactory.createCacheKey(operationContent: "perf-test-\(i)") + let result = TestDataFactory.createCachedResult(snapshotContent: "content-\(i)") + await cache.put(result, key: key, for: operation) + _ = await cache.get(key, for: operation) + } + } + + // Assert reasonable performance (adjust threshold as needed) + #expect(duration < 5.0, "Cache operations took too long: \(duration)s") + } + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildExecutorTests/ExecutionContextTests.swift b/Tests/NativeBuilderTests/ContainerBuildExecutorTests/ExecutionContextTests.swift new file mode 100644 index 00000000..6c95543c --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildExecutorTests/ExecutionContextTests.swift @@ -0,0 +1,188 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildReporting +import ContainerBuildSnapshotter +import Foundation +import Testing + +@testable import ContainerBuildExecutor + +struct ExecutionContextTests { + + @Test func contextStateManagement() async throws { + let stage = BuildStage( + id: UUID(), + name: "test", + base: ImageOperation( + source: .scratch, + platform: nil, + pullPolicy: .ifNotPresent, + verification: nil, + metadata: OperationMetadata() + ), + nodes: [], + platform: nil + ) + + let graph = try BuildGraph( + stages: [stage], + buildArgs: [:], + targetPlatforms: [.linuxAMD64], + metadata: BuildGraphMetadata() + ) + + let context = ExecutionContext( + stage: stage, + graph: graph, + platform: .linuxAMD64, + reporter: Reporter() + ) + + // Test environment management + #expect(context.environment.variables.isEmpty == true) + + context.updateEnvironment(["FOO": EnvironmentValue.literal("bar")]) + #expect(context.environment.get("FOO") == EnvironmentValue.literal("bar")) + + // Test working directory + #expect(context.workingDirectory == "/") + context.setWorkingDirectory("/app") + #expect(context.workingDirectory == "/app") + + // Test user + #expect(context.user == nil) + let user = User.userGroup(user: "appuser", group: "appgroup") + context.setUser(user) + #expect(context.user == user) + + // Test snapshots + let snapshot = Snapshot( + digest: try! Digest(algorithm: .sha256, bytes: Data(count: 32)), + size: 1024 + ) + let nodeId = UUID() + context.setSnapshot(snapshot, for: nodeId) + #expect(context.snapshot(for: nodeId)?.id == snapshot.id) + #expect(context.latestSnapshot() != nil) + } + + @Test func imageConfigUpdates() async throws { + let stage = BuildStage( + id: UUID(), + name: "test", + base: ImageOperation( + source: .scratch, + platform: nil, + pullPolicy: .ifNotPresent, + verification: nil, + metadata: OperationMetadata() + ), + nodes: [], + platform: nil + ) + + let graph = try BuildGraph( + stages: [stage], + buildArgs: [:], + targetPlatforms: [.linuxAMD64], + metadata: BuildGraphMetadata() + ) + + let context = ExecutionContext( + stage: stage, + graph: graph, + platform: .linuxAMD64, + reporter: Reporter() + ) + + // Update image config + context.updateImageConfig { config in + config.env = ["PATH=/usr/bin"] + config.cmd = ["echo", "hello"] + config.workingDir = "/app" + config.exposedPorts.insert("8080/tcp") + config.labels["version"] = "1.0" + } + + let config = context.imageConfig + #expect(config.env == ["PATH=/usr/bin"]) + #expect(config.cmd == ["echo", "hello"]) + #expect(config.workingDir == "/app") + #expect(config.exposedPorts.contains("8080/tcp") == true) + #expect(config.labels["version"] == "1.0") + } + + @Test func childContext() async throws { + let stage1 = BuildStage( + id: UUID(), + name: "stage1", + base: ImageOperation( + source: .scratch, + platform: nil, + pullPolicy: .ifNotPresent, + verification: nil, + metadata: OperationMetadata() + ), + nodes: [], + platform: nil + ) + + let stage2 = BuildStage( + id: UUID(), + name: "stage2", + base: ImageOperation( + source: .scratch, + platform: nil, + pullPolicy: .ifNotPresent, + verification: nil, + metadata: OperationMetadata() + ), + nodes: [], + platform: nil + ) + + let graph = try BuildGraph( + stages: [stage1, stage2], + buildArgs: [:], + targetPlatforms: [.linuxAMD64], + metadata: BuildGraphMetadata() + ) + + let parentContext = ExecutionContext( + stage: stage1, + graph: graph, + platform: .linuxAMD64, + reporter: Reporter() + ) + + // Set up parent context + parentContext.updateEnvironment(["PARENT": EnvironmentValue.literal("value")]) + parentContext.setWorkingDirectory("/parent") + + // Create child context + let childContext = parentContext.childContext(for: stage2) + + // Child should inherit environment + #expect(childContext.environment.get("PARENT") == EnvironmentValue.literal("value")) + + // But modifications to child don't affect parent + childContext.updateEnvironment(["CHILD": EnvironmentValue.literal("value")]) + #expect(parentContext.environment.get("CHILD") == nil) + #expect(childContext.environment.get("CHILD") == EnvironmentValue.literal("value")) + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildExecutorTests/ExecutionDispatcherTests.swift b/Tests/NativeBuilderTests/ContainerBuildExecutorTests/ExecutionDispatcherTests.swift new file mode 100644 index 00000000..8e28381d --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildExecutorTests/ExecutionDispatcherTests.swift @@ -0,0 +1,250 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildReporting +import ContainerBuildSnapshotter +import Foundation +import Testing + +@testable import ContainerBuildExecutor + +struct ExecutionDispatcherTests { + + @Test func dispatcherRouting() async throws { + // Create test executors with different capabilities + let execExecutor = ExecOperationExecutor() + let fsExecutor = FilesystemOperationExecutor() + let metadataExecutor = MetadataOperationExecutor() + + let dispatcher = ExecutionDispatcher(executors: [ + execExecutor, + fsExecutor, + metadataExecutor, + ]) + + // Create test context + let context = try createTestContext() + + // Test exec operation routing + let execOp = ExecOperation( + command: .shell("echo test"), + environment: .empty, + mounts: [], + workingDirectory: nil, + user: nil, + network: .default, + security: .default, + metadata: OperationMetadata() + ) + + let execResult = try await dispatcher.dispatch(execOp, context: context) + #expect(execResult.duration > 0) + + // Test filesystem operation routing + let fsOp = FilesystemOperation( + action: .copy, + source: .context(ContextSource(paths: ["test.txt"])), + destination: "/app/test.txt", + fileMetadata: FileMetadata(), + options: FilesystemOptions(), + metadata: OperationMetadata() + ) + + let fsResult = try await dispatcher.dispatch(fsOp, context: context) + #expect(fsResult.filesystemChanges.added.contains("/app/test.txt") == true) + + // Test metadata operation routing + let metadataOp = MetadataOperation( + action: .setEnv(key: "TEST", value: .literal("value")), + metadata: OperationMetadata() + ) + + let metadataResult = try await dispatcher.dispatch(metadataOp, context: context) + #expect(metadataResult.environmentChanges["TEST"] == EnvironmentValue.literal("value")) + } + + @Test func capabilityMatching() async throws { + // Create a custom executor with specific capabilities + struct PrivilegedExecutor: OperationExecutor { + let capabilities = ExecutorCapabilities( + supportedOperations: [.exec], + requiresPrivileged: true, + maxConcurrency: 1 + ) + + func execute(_ operation: ContainerBuildIR.Operation, context: ExecutionContext) async throws -> ExecutionResult { + let digest = try! Digest(algorithm: .sha256, bytes: Data(count: 32)) + let snapshot = Snapshot(digest: digest, size: 0) + return ExecutionResult( + snapshot: snapshot, + duration: 0.1 + ) + } + + func canExecute(_ operation: ContainerBuildIR.Operation) -> Bool { + operation is ExecOperation + } + } + + let regularExecutor = ExecOperationExecutor() + let privilegedExecutor = PrivilegedExecutor() + + let dispatcher = ExecutionDispatcher(executors: [ + regularExecutor, + privilegedExecutor, + ]) + + let context = try createTestContext() + let execOp = ExecOperation( + command: .shell("privileged command"), + environment: .empty, + mounts: [], + workingDirectory: nil, + user: nil, + network: .default, + security: .default, + metadata: OperationMetadata() + ) + + // Without constraints, should use regular executor + _ = try await dispatcher.dispatch(execOp, context: context) + + // With privileged constraint, should use privileged executor + let constraints = NodeConstraints(requiresPrivileged: true) + _ = try await dispatcher.dispatch( + execOp, + context: context, + constraints: constraints + ) + } + + @Test func unsupportedOperation() async throws { + // Create dispatcher with limited executors + let dispatcher = ExecutionDispatcher(executors: [ + ExecOperationExecutor() + ]) + + let context = try createTestContext() + + // Try to dispatch an unsupported operation + struct CustomOperation: ContainerBuildIR.Operation { + static let operationKind = OperationKind(rawValue: "custom") + var operationKind: OperationKind { Self.operationKind } + let metadata: OperationMetadata = OperationMetadata() + + func accept(_ visitor: V) throws -> V.Result { + try visitor.visitUnknown(self) + } + } + + let customOp = CustomOperation() + + await #expect(throws: (any Error).self) { + try await dispatcher.dispatch(customOp, context: context) + } + } + + @Test func concurrencyLimiting() async throws { + // Create executor with limited concurrency + struct SlowExecutor: OperationExecutor { + let capabilities = ExecutorCapabilities( + supportedOperations: [.exec], + maxConcurrency: 2 + ) + + func execute(_ operation: ContainerBuildIR.Operation, context: ExecutionContext) async throws -> ExecutionResult { + // Simulate slow operation + try await Task.sleep(nanoseconds: 100_000_000) // 100ms + let digest = try! Digest(algorithm: .sha256, bytes: Data(count: 32)) + let snapshot = Snapshot(digest: digest, size: 0) + return ExecutionResult( + snapshot: snapshot, + duration: 0.1 + ) + } + + func canExecute(_ operation: ContainerBuildIR.Operation) -> Bool { + operation is ExecOperation + } + } + + let dispatcher = ExecutionDispatcher(executors: [SlowExecutor()]) + let context = try createTestContext() + + // Create multiple operations + let operations = (0..<5).map { i in + ExecOperation( + command: .shell("echo \(i)"), + environment: .empty, + mounts: [], + workingDirectory: nil, + user: nil, + network: .default, + security: .default, + metadata: OperationMetadata() + ) + } + + // Dispatch all operations concurrently + let startTime = Date() + try await withThrowingTaskGroup(of: Void.self) { group in + for op in operations { + group.addTask { + _ = try await dispatcher.dispatch(op, context: context) + } + } + try await group.waitForAll() + } + let duration = Date().timeIntervalSince(startTime) + + // With max concurrency 2 and 5 operations at 100ms each, + // should take at least 300ms (3 batches) + #expect(duration > 0.25) + } + + // MARK: - Helpers + + private func createTestContext() throws -> ExecutionContext { + let stage = BuildStage( + id: UUID(), + name: "test", + base: ImageOperation( + source: .scratch, + platform: nil, + pullPolicy: .ifNotPresent, + verification: nil, + metadata: OperationMetadata() + ), + nodes: [], + platform: nil + ) + + let graph = try BuildGraph( + stages: [stage], + buildArgs: [:], + targetPlatforms: [.linuxAMD64], + metadata: BuildGraphMetadata() + ) + + return ExecutionContext( + stage: stage, + graph: graph, + platform: .linuxAMD64, + reporter: Reporter() + ) + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildExecutorTests/ReporterTests.swift b/Tests/NativeBuilderTests/ContainerBuildExecutorTests/ReporterTests.swift new file mode 100644 index 00000000..6daffdfc --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildExecutorTests/ReporterTests.swift @@ -0,0 +1,212 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerBuildReporting +import Foundation +import Testing + +@testable import ContainerBuildExecutor + +@MainActor +final class EventCollector { + private var events: [BuildEvent] = [] + + func add(_ event: BuildEvent) { + events.append(event) + } + + func getEvents() -> [BuildEvent] { + events + } +} + +struct ReporterTests { + @Test func reporterEmitsEvents() async throws { + let reporter = Reporter() + let eventCollector = EventCollector() + + // Collect events in background + let collectionTask = Task { + for await event in reporter.stream { + await eventCollector.add(event) + } + } + + // Emit test events + await reporter.report(.buildStarted(totalOperations: 5, stages: 2, timestamp: Date())) + + let context = ReportContext( + nodeId: UUID(), + stageId: "test-stage", + description: "RUN echo hello" + ) + + await reporter.report(.operationStarted(context: context)) + await reporter.report(.operationLog(context: context, message: "Hello, world!")) + await reporter.report(.operationFinished(context: context, duration: 1.5)) + + await reporter.report(.buildCompleted(success: true, timestamp: Date())) + await reporter.finish() + + // Wait for collection to complete + await collectionTask.value + + // Verify events + let events = await eventCollector.getEvents() + #expect(events.count == 5) + + if case .buildStarted(let ops, let stages, _) = events[0] { + #expect(ops == 5) + #expect(stages == 2) + } else { + Issue.record("Expected buildStarted event") + } + + if case .operationStarted(let ctx) = events[1] { + #expect(ctx.description == "RUN echo hello") + } else { + Issue.record("Expected operationStarted event") + } + + if case .operationLog(_, let message) = events[2] { + #expect(message == "Hello, world!") + } else { + Issue.record("Expected operationLog event") + } + + if case .operationFinished(_, let duration) = events[3] { + #expect(duration == 1.5) + } else { + Issue.record("Expected operationFinished event") + } + + if case .buildCompleted(let success, _) = events[4] { + #expect(success == true) + } else { + Issue.record("Expected buildCompleted event") + } + } + + @Test func plainProgressConsumer() async throws { + let reporter = Reporter() + + // Start consumer in background + let consumerTask = Task { + let consumer = PlainProgressConsumer(configuration: .init()) + try await consumer.consume(reporter: reporter) + } + + // Emit events + await reporter.report(.buildStarted(totalOperations: 2, stages: 1, timestamp: Date())) + + let nodeId = UUID() + let context = ReportContext( + nodeId: nodeId, + stageId: "main", + description: "RUN apt-get update" + ) + + await reporter.report(.stageStarted(stageName: "main", timestamp: Date())) + await reporter.report(.operationStarted(context: context)) + await reporter.report(.operationLog(context: context, message: "Reading package lists...")) + await reporter.report(.operationLog(context: context, message: "Building dependency tree...")) + await reporter.report(.operationFinished(context: context, duration: 2.3)) + await reporter.report(.stageCompleted(stageName: "main", timestamp: Date())) + await reporter.report(.buildCompleted(success: true, timestamp: Date())) + + await reporter.finish() + try await consumerTask.value + + // Test passes if consumer completes without error + } + + @Test func operationCacheHit() async throws { + let reporter = Reporter() + let eventCollector = EventCollector() + + let collectionTask = Task { + for await event in reporter.stream { + await eventCollector.add(event) + } + } + + let context = ReportContext( + nodeId: UUID(), + stageId: "cached-stage", + description: "COPY src/ /app/" + ) + + await reporter.report(.operationStarted(context: context)) + await reporter.report(.operationCacheHit(context: context)) + await reporter.finish() + + await collectionTask.value + + let events = await eventCollector.getEvents() + #expect(events.count == 2) + + if case .operationCacheHit(let ctx) = events[1] { + #expect(ctx.description == "COPY src/ /app/") + } else { + Issue.record("Expected operationCacheHit event") + } + } + + @Test func operationFailure() async throws { + let reporter = Reporter() + let eventCollector = EventCollector() + + let collectionTask = Task { + for await event in reporter.stream { + await eventCollector.add(event) + } + } + + let context = ReportContext( + nodeId: UUID(), + stageId: "failing-stage", + description: "RUN false" + ) + + let error = BuildEventError( + type: .executionFailed, + description: "Command exited with non-zero status", + diagnostics: [ + "exitCode": "1", + "workingDirectory": "/app", + ] + ) + + await reporter.report(.operationStarted(context: context)) + await reporter.report(.operationFailed(context: context, error: error)) + await reporter.report(.buildCompleted(success: false, timestamp: Date())) + await reporter.finish() + + await collectionTask.value + + let events = await eventCollector.getEvents() + #expect(events.count == 3) + + if case .operationFailed(let ctx, let err) = events[1] { + #expect(ctx.description == "RUN false") + #expect(err.type == .executionFailed) + #expect(err.diagnostics?["exitCode"] == "1") + } else { + Issue.record("Expected operationFailed event") + } + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildExecutorTests/SimpleExecutorTests.swift b/Tests/NativeBuilderTests/ContainerBuildExecutorTests/SimpleExecutorTests.swift new file mode 100644 index 00000000..edb037be --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildExecutorTests/SimpleExecutorTests.swift @@ -0,0 +1,152 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildCache +import ContainerBuildIR +import Testing + +@testable import ContainerBuildExecutor + +struct SimpleExecutorTests { + + @Test func simpleBuildExecution() async throws { + // Create a simple build graph + let imageRef = ImageReference(parsing: "ubuntu:22.04")! + let graph = try GraphBuilder.singleStage( + from: imageRef, + platform: .linuxAMD64 + ) { builder in + try builder + .run("apt-get update") + .run("apt-get install -y curl") + .workdir("/app") + .copy(from: .context(ContextSource(paths: ["main.go"])), to: "/app/") + .run("go build -o app main.go") + .cmd(Command.exec(["./app"])) + } + + // Create executor (using Scheduler as the main executor) + let executor = Scheduler() + + // Execute the build + let result = try await executor.execute(graph) + + // Verify results + #expect(result.manifests.count == 1) + #expect(result.manifests[.linuxAMD64] != nil) + #expect(result.metrics.operationCount > 0) + #expect(result.metrics.totalDuration >= 0) + } + + @Test func multiStageBuildExecution() async throws { + // Create a multi-stage build + let builderImageRef = ImageReference(parsing: "golang:1.21")! + let alpineImageRef = ImageReference(parsing: "alpine:latest")! + let graph = try GraphBuilder.multiStage { builder in + // Build stage + try builder + .stage(name: "builder", from: builderImageRef) + .workdir("/src") + .copy(from: .context(ContextSource(paths: ["go.mod", "go.sum"])), to: "./") + .run("go mod download") + .copy(from: .context(ContextSource(paths: ["*.go"])), to: "./") + .run("go build -o /app") + + // Runtime stage + try builder + .stage(from: alpineImageRef) + .copy(from: .stage(.named("builder"), paths: ["/app"]), to: "/usr/local/bin/") + .entrypoint(Command.exec(["/usr/local/bin/app"])) + } + + let executor = Scheduler() + let result = try await executor.execute(graph) + + #expect(result.manifests.count == 1) + #expect(result.metrics.operationCount >= 2) + } + + @Test func cancellation() async throws { + // Create a graph with many operations + let imageRef = ImageReference(parsing: "ubuntu:22.04")! + let graph = try GraphBuilder.singleStage( + from: imageRef + ) { builder in + for i in 0..<100 { + try builder.run("echo Step \(i)") + } + } + + let executor = Scheduler() + + // Start execution and cancel immediately + Task { + try? await Task.sleep(nanoseconds: 10_000_000) // 10ms + await executor.cancel() + } + + await #expect(throws: (any Error).self) { + try await executor.execute(graph) + } + } + + @Test func caching() async throws { + let cache = MemoryBuildCache() + let executor = Scheduler(cache: cache) + + let imageRef = ImageReference(parsing: "alpine:latest")! + let graph = try GraphBuilder.singleStage( + from: imageRef + ) { builder in + try builder.run("echo 'Hello, World!'") + } + + // First execution + let result1 = try await executor.execute(graph) + #expect(result1.metrics.cachedOperationCount == 0) + + // Second execution should use cache + let result2 = try await executor.execute(graph) + #expect(result2.metrics.cachedOperationCount > 0) + + // Verify cache stats + let stats = await cache.statistics() + #expect(stats.hitRate > 0) + } + + @Test func executorCapabilities() async throws { + // Test that operations are routed to correct executors + let execExecutor = ExecOperationExecutor() + let fsExecutor = FilesystemOperationExecutor() + + #expect(execExecutor.capabilities.supportedOperations.contains(.exec) == true) + #expect(fsExecutor.capabilities.supportedOperations.contains(.filesystem) == true) + + let execOp = ExecOperation( + command: .shell("echo test"), + environment: .empty, + mounts: [], + workingDirectory: nil, + user: nil, + network: .default, + security: .default, + metadata: OperationMetadata() + ) + + #expect(execExecutor.canExecute(execOp) == true) + #expect(fsExecutor.canExecute(execOp) == false) + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildIRTests/AnalysisTests.swift b/Tests/NativeBuilderTests/ContainerBuildIRTests/AnalysisTests.swift new file mode 100644 index 00000000..05e836c3 --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildIRTests/AnalysisTests.swift @@ -0,0 +1,612 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation +import Testing + +@testable import ContainerBuildIR +@testable import ContainerBuildReporting + +struct AnalysisTests { + + // MARK: - SemanticAnalyzer Integration Tests + + @Test func semanticAnalyzerGraphAnalyzerProtocol() throws { + guard let alpineRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + let originalGraph = try GraphBuilder.singleStage(from: alpineRef) { builder in + try builder + .run("apk add --no-cache curl") + .workdir("/app") + .copyFromContext(paths: ["src/"], to: "/app/") + .user(.uid(1000)) + .entrypoint(.exec(["/app/server"])) + } + + let analyzer = SemanticAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let analyzedGraph = try analyzer.analyze(originalGraph, context: context) + + // SemanticAnalyzer should return the graph unchanged + #expect(analyzedGraph.stages.count == originalGraph.stages.count) + #expect(analyzedGraph.buildArgs == originalGraph.buildArgs) + #expect(analyzedGraph.targetPlatforms == originalGraph.targetPlatforms) + } + + @Test func semanticAnalyzerWithReporter() async throws { + guard let ubuntuRef = ImageReference(parsing: "ubuntu:22.04") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a graph with potential issues + let graph = try GraphBuilder.singleStage(from: ubuntuRef) { builder in + try builder + .run("apt-get update") // Separate update (should trigger layer warning) + .run("apt-get install -y curl") // Separate install (should trigger layer warning) + .run("wget https://example.com/script.sh") // Unverified download (security warning) + .workdir("/app") + .copyFromContext(paths: ["src/"], to: "/app/") + // No USER instruction (should trigger security warning) + } + + let reporter = Reporter() + let context = AnalysisContext(reporter: reporter, sourceMap: nil) + + let analyzer = SemanticAnalyzer() + let _ = try analyzer.analyze(graph, context: context) + + // Allow some time for async reporting + try await Task.sleep(nanoseconds: 100_000_000) // 0.1 second + + // Verify that some events were reported + // Note: We can't easily test the exact events without implementing a test reporter + // that captures events, but we can verify the analyze method completed successfully + #expect(Bool(true)) // Analysis completed without throwing + } + + // MARK: - Layer Efficiency Analysis Tests + + @Test func layerEfficiencyMultiplePackageManagers() throws { + guard let ubuntuRef = ImageReference(parsing: "ubuntu:22.04") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a graph with multiple package manager calls + let graph = try GraphBuilder.singleStage(from: ubuntuRef) { builder in + try builder + .run("apt-get update") + .run("apt-get install -y curl") + .run("apt-get install -y wget") + .run("apt-get install -y git") + } + + let analyzer = SemanticAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + // This should trigger layer efficiency warnings + let _ = try analyzer.analyze(graph, context: context) + + // Verify analysis completed successfully + #expect(Bool(true)) + } + + @Test func layerEfficiencyAddThenRemove() throws { + guard let alpineRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a graph that adds then removes files + let graph = try GraphBuilder.singleStage(from: alpineRef) { builder in + try builder + .run("apk add --no-cache build-base") + .copyFromContext(paths: ["src/"], to: "/tmp/build/") + .run("cd /tmp/build && make") + .run("rm -rf /tmp/build") // Remove build files + } + + let analyzer = SemanticAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + // This should trigger layer efficiency warnings about unnecessary files + let _ = try analyzer.analyze(graph, context: context) + + #expect(Bool(true)) + } + + // MARK: - Security Analysis Tests + + @Test func securityAnalysisRootUser() throws { + guard let alpineRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a graph that runs as root (no USER instruction) + let graph = try GraphBuilder.singleStage(from: alpineRef) { builder in + try builder + .run("apk add --no-cache curl") + .workdir("/app") + .copyFromContext(paths: ["app"], to: "/app/") + .entrypoint(.exec(["/app/app"])) + // No USER instruction - should trigger security warning + } + + let analyzer = SemanticAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let _ = try analyzer.analyze(graph, context: context) + + #expect(Bool(true)) + } + + @Test func securityAnalysisPrivilegedExecution() throws { + guard let alpineRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a graph with privileged execution + let graph = try GraphBuilder.singleStage(from: alpineRef) { builder in + try builder + .run("mount /dev/sda1 /mnt") + .workdir("/app") + .user(.uid(1000)) + } + + let analyzer = SemanticAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let _ = try analyzer.analyze(graph, context: context) + + #expect(Bool(true)) + } + + @Test func securityAnalysisUnverifiedDownloads() throws { + guard let alpineRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a graph with unverified downloads + let graph = try GraphBuilder.singleStage(from: alpineRef) { builder in + try builder + .run("wget https://example.com/install.sh && sh install.sh") + .run("curl -sSL https://get.docker.com | sh") + .workdir("/app") + .user(.uid(1000)) + } + + let analyzer = SemanticAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let _ = try analyzer.analyze(graph, context: context) + + #expect(Bool(true)) + } + + @Test func securityAnalysisUnpinnedDependencies() throws { + guard let ubuntuRef = ImageReference(parsing: "ubuntu:22.04") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a graph with unpinned dependencies + let graph = try GraphBuilder.singleStage(from: ubuntuRef) { builder in + try builder + .run("apt-get update") + .run("apt-get install -y curl") // No version pinning + .run("pip install flask") // No version pinning + .workdir("/app") + .user(.uid(1000)) + } + + let analyzer = SemanticAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let _ = try analyzer.analyze(graph, context: context) + + #expect(Bool(true)) + } + + // MARK: - Cache Analysis Tests + + @Test func cacheAnalysisTimestampInvalidation() throws { + guard let alpineRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a graph with timestamp-based cache invalidation + let graph = try GraphBuilder.singleStage(from: alpineRef) { builder in + try builder + .run("apk add --no-cache curl") + .run("echo $(date) > /app/build-time.txt") // Timestamp invalidates cache + .workdir("/app") + .user(.uid(1000)) + } + + let analyzer = SemanticAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let _ = try analyzer.analyze(graph, context: context) + + #expect(Bool(true)) + } + + @Test func cacheAnalysisRandomInvalidation() throws { + guard let alpineRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a graph with random data cache invalidation + let graph = try GraphBuilder.singleStage(from: alpineRef) { builder in + try builder + .run("apk add --no-cache curl") + .run("echo $RANDOM > /app/random.txt") // Random invalidates cache + .run("openssl rand -hex 16 > /app/key.txt") // Random invalidates cache + .workdir("/app") + .user(.uid(1000)) + } + + let analyzer = SemanticAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let _ = try analyzer.analyze(graph, context: context) + + #expect(Bool(true)) + } + + // MARK: - Size Optimization Tests + + @Test func sizeOptimizationPackageCache() throws { + guard let ubuntuRef = ImageReference(parsing: "ubuntu:22.04") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a graph without package cache cleanup + let graph = try GraphBuilder.singleStage(from: ubuntuRef) { builder in + try builder + .run("apt-get update") + .run("apt-get install -y curl wget git") + // No cleanup - should trigger size optimization warning + .workdir("/app") + .user(.uid(1000)) + } + + let analyzer = SemanticAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let _ = try analyzer.analyze(graph, context: context) + + #expect(Bool(true)) + } + + @Test func sizeOptimizationBuildDependencies() throws { + guard let alpineRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a single-stage graph with build dependencies + let graph = try GraphBuilder.singleStage(from: alpineRef) { builder in + try builder + .run("apk add --no-cache build-base gcc-dev") // Build dependencies + .workdir("/app") + .copyFromContext(paths: ["src/"], to: "/app/") + .run("make") + .user(.uid(1000)) + .entrypoint(.exec(["/app/server"])) + } + + let analyzer = SemanticAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let _ = try analyzer.analyze(graph, context: context) + + #expect(Bool(true)) + } + + @Test func sizeOptimizationMultiStageComparison() throws { + guard let alpineRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a multi-stage graph (should NOT trigger build dependency warnings) + let graph = try GraphBuilder.multiStage { builder in + try builder + .stage(name: "builder", from: alpineRef) + .run("apk add --no-cache build-base") + .workdir("/app") + .copyFromContext(paths: ["src/"], to: "/app/") + .run("make") + + try builder + .stage(from: alpineRef) + .copyFromStage(.named("builder"), paths: ["/app/server"], to: "/app/") + .user(.uid(1000)) + .entrypoint(.exec(["/app/server"])) + } + + let analyzer = SemanticAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let _ = try analyzer.analyze(graph, context: context) + + #expect(Bool(true)) + } + + // MARK: - Custom Analyzer Tests + + struct CustomSecurityAnalyzer: GraphAnalyzer { + func analyze(_ graph: BuildGraph, context: AnalysisContext) throws -> BuildGraph { + // Custom analysis: Check for hardcoded secrets + for stage in graph.stages { + for node in stage.nodes { + if let exec = node.operation as? ExecOperation { + if case .shell(let cmd) = exec.command { + if cmd.contains("PASSWORD=") || cmd.contains("SECRET=") { + if let reporter = context.reporter { + Task { + await reporter.report( + .irEvent( + context: ReportContext( + description: "Potential hardcoded secret detected in command", + sourceMap: nil + ), + type: .error + )) + } + } + } + } + } + } + } + + return graph + } + } + + @Test func customAnalyzerExtensibility() throws { + guard let alpineRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a graph with hardcoded secrets + let graph = try GraphBuilder.singleStage(from: alpineRef) { builder in + try builder + .run("export PASSWORD=secret123") // Hardcoded secret + .run("SECRET=api_key_123 ./app") // Hardcoded secret + .workdir("/app") + .user(.uid(1000)) + } + + let customAnalyzer = CustomSecurityAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let _ = try customAnalyzer.analyze(graph, context: context) + + #expect(Bool(true)) + } + + // MARK: - Analyzer Chain Tests + + @Test func analyzerChain() throws { + guard let alpineRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + let graph = try GraphBuilder.singleStage(from: alpineRef) { builder in + try builder + .run("apk add --no-cache curl") + .workdir("/app") + .copyFromContext(paths: ["src/"], to: "/app/") + .user(.uid(1000)) + .entrypoint(.exec(["/app/server"])) + } + + let analyzers: [any GraphAnalyzer] = [ + DependencyAnalyzer(), + SemanticAnalyzer(), + CustomSecurityAnalyzer(), + ] + + var currentGraph = graph + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + // Apply analyzers in sequence + for analyzer in analyzers { + currentGraph = try analyzer.analyze(currentGraph, context: context) + } + + // Graph should be preserved through the chain + #expect(currentGraph.stages.count == graph.stages.count) + #expect(currentGraph.buildArgs == graph.buildArgs) + } + + // MARK: - Performance Tests + + @Test func analysisPerformanceLargeGraph() throws { + guard let alpineRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a large graph for performance testing + let graph = try GraphBuilder.multiStage { builder in + for stageIndex in 0..<10 { + try builder + .stage(name: "stage\(stageIndex)", from: alpineRef) + .run("apk add --no-cache curl") + .run("apk add --no-cache wget") + .run("apk add --no-cache git") + .workdir("/app") + .copyFromContext(paths: ["file\(stageIndex).txt"], to: "/app/") + .env("STAGE", "\(stageIndex)") + .label("stage.number", "\(stageIndex)") + .user(.uid(1000)) + } + } + + let analyzer = SemanticAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let startTime = Date() + let _ = try analyzer.analyze(graph, context: context) + let duration = Date().timeIntervalSince(startTime) + + print("Semantic analysis for large graph (10 stages): \(String(format: "%.3f", duration))s") + #expect(duration < 1.0, "Analysis should complete quickly for large graphs") + } + + // MARK: - Error Handling Tests + + @Test func analysisErrorHandling() throws { + // Test that analysis handles malformed operations gracefully + guard let alpineRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + // GraphBuilder will throw validation errors for malformed operations + #expect(throws: ValidationError.self) { + try GraphBuilder.singleStage(from: alpineRef) { builder in + try builder + .run("") // Empty command + .workdir("") // Empty workdir + .copyFromContext(paths: [], to: "") // Empty paths and destination + .user(.uid(0)) // Root user + } + } + } + + // MARK: - Reporting Integration Tests + + @Test func analysisReportingIntegration() async throws { + guard let ubuntuRef = ImageReference(parsing: "ubuntu:22.04") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a graph with multiple types of issues + let graph = try GraphBuilder.singleStage(from: ubuntuRef) { builder in + try builder + .run("apt-get update") // Layer efficiency issue + .run("apt-get install -y curl") // Layer efficiency issue + .run("apt-get install -y wget") // Layer efficiency issue + .run("wget https://example.com/script.sh") // Security issue + .run("echo $RANDOM > /app/random.txt") // Cache invalidation issue + .workdir("/app") + // No USER instruction - security issue + // No cleanup - size optimization issue + } + + let reporter = Reporter() + let context = AnalysisContext( + reporter: reporter, + sourceMap: SourceMap( + file: "Dockerfile", + line: 1, + column: 1, + snippet: "FROM ubuntu:22.04" + ) + ) + + let analyzer = SemanticAnalyzer() + let _ = try analyzer.analyze(graph, context: context) + + // Allow time for async reporting + try await Task.sleep(nanoseconds: 100_000_000) // 0.1 second + + #expect(Bool(true)) // Analysis and reporting completed successfully + } + + // MARK: - Real-world Scenario Tests + + @Test func analysisNodeJSApplication() throws { + guard let nodeRef = ImageReference(parsing: "node:18-alpine") else { + Issue.record("Failed to parse image reference") + return + } + + // Realistic Node.js application build + let graph = try GraphBuilder.singleStage(from: nodeRef) { builder in + try builder + .run("apk add --no-cache dumb-init") // Good: single package install + .workdir("/app") + .copyFromContext(paths: ["package*.json"], to: "./") + .run("npm ci --only=production && npm cache clean --force") // Good: cleanup + .copyFromContext(paths: ["src/"], to: "./src/") + .user(.uid(1000)) // Good: non-root user + .expose(3000) + .entrypoint(.exec(["dumb-init", "node", "src/index.js"])) + } + + let analyzer = SemanticAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let _ = try analyzer.analyze(graph, context: context) + + #expect(Bool(true)) + } + + @Test func analysisGoApplication() throws { + guard let golangRef = ImageReference(parsing: "golang:1.21-alpine"), + let alpineRef = ImageReference(parsing: "alpine:latest") + else { + Issue.record("Failed to parse image references") + return + } + + // Realistic Go multi-stage build + let graph = try GraphBuilder.multiStage { builder in + try builder + .stage(name: "builder", from: golangRef) + .workdir("/src") + .copyFromContext(paths: ["go.mod", "go.sum"], to: "./") + .run("go mod download") + .copyFromContext(paths: [".", "!**/*_test.go"], to: "./") + .run("CGO_ENABLED=0 GOOS=linux go build -ldflags='-w -s' -o /app cmd/main.go") + + try builder + .stage(from: alpineRef) + .run("apk add --no-cache ca-certificates && rm -rf /var/cache/apk/*") // Good: cleanup + .copyFromStage(.named("builder"), paths: ["/app"], to: "/usr/local/bin/") + .user(.uid(65534)) // Good: nobody user + .expose(8080) + .entrypoint(.exec(["/usr/local/bin/app"])) + } + + let analyzer = SemanticAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let _ = try analyzer.analyze(graph, context: context) + + #expect(Bool(true)) + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildIRTests/BasicTests.swift b/Tests/NativeBuilderTests/ContainerBuildIRTests/BasicTests.swift new file mode 100644 index 00000000..bc5cc8fb --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildIRTests/BasicTests.swift @@ -0,0 +1,315 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOCI +import Foundation +import Testing + +@testable import ContainerBuildIR + +struct BasicTests { + + @Test func digestCreation() throws { + // Test creating digest from bytes + let bytes = Data(repeating: 0xAB, count: 32) + let digest = try Digest(algorithm: .sha256, bytes: bytes) + #expect(digest.algorithm == .sha256) + #expect(digest.bytes == bytes) + + // Test parsing digest string + let digestString = "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" + let parsed = try Digest(parsing: digestString) + #expect(parsed.stringValue == digestString) + + // Test invalid length + #expect(throws: Error.self) { + try Digest(algorithm: .sha256, bytes: Data(count: 16)) + } + } + + @Test func imageReference() throws { + // Test parsing various formats + let refs = [ + ("ubuntu", "ubuntu:latest"), + ("ubuntu:20.04", "ubuntu:20.04"), + ("ghcr.io/owner/repo:tag", "ghcr.io/owner/repo:tag"), + ("localhost:5000/test", "localhost:5000/test:latest"), + ("image@sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", "image@sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"), + ] + + for (input, expected) in refs { + guard let ref = ImageReference(parsing: input) else { + Issue.record("Failed to parse: \(input)") + continue + } + #expect(ref.stringValue == expected, "Failed for input: \(input)") + } + + // Test creation + let ref = try ImageReference( + registry: "docker.io", + repository: "library/nginx", + tag: "alpine" + ) + #expect(ref.stringValue == "docker.io/library/nginx:alpine") + } + + @Test func simpleGraph() throws { + // Create a simple graph + guard let imageRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + let graph = try GraphBuilder.singleStage( + from: imageRef + ) { builder in + try builder + .run("apk add --no-cache curl") + .workdir("/app") + .copyFromContext(paths: ["main.go"], to: "/app/") + .run("go build -o app main.go") + .entrypoint(.exec(["/app/app"])) + } + + #expect(graph.stages.count == 1) + #expect(graph.stages[0].nodes.count == 5) + + // Validate the graph + let validator = StandardValidator() + let result = validator.validate(graph) + #expect(result.isValid == true, "Graph validation failed: \(result.errors)") + } + + @Test func multiStageGraph() throws { + guard let golangRef = ImageReference(parsing: "golang:1.21"), + let alpineRef = ImageReference(parsing: "alpine:latest") + else { + Issue.record("Failed to parse image references") + return + } + + let graph = try GraphBuilder.multiStage { builder in + // Build stage + try builder + .stage(name: "builder", from: golangRef) + .workdir("/src") + .copyFromContext(paths: ["go.mod", "go.sum", "*.go"], to: "./") + .run("go build -o /app") + + // Runtime stage + try builder + .stage(from: alpineRef) + .run("apk add --no-cache ca-certificates") + .copyFromStage(.named("builder"), paths: ["/app"], to: "/usr/local/bin/app") + .user(.uid(1000)) + .entrypoint(Command.exec(["/usr/local/bin/app"])) + } + + #expect(graph.stages.count == 2) + #expect(graph.stages[0].name == "builder") + #expect(graph.stages[1].name == nil) + + // Check stage dependencies + let deps = graph.stages[1].stageDependencies() + #expect(deps.contains(.named("builder")) == true) + } + + @Test func operationTypes() throws { + // Test ExecOperation + let execOp = ExecOperation( + command: .shell("echo 'Hello, World!'"), + environment: Environment([ + (key: "FOO", value: .literal("bar")) + ]), + workingDirectory: "/tmp" + ) + #expect(execOp.command.displayString == "echo 'Hello, World!'") + #expect(execOp.environment.effectiveEnvironment["FOO"] == "bar") + + // Test FilesystemOperation + let fsOp = FilesystemOperation( + action: .copy, + source: .context(ContextSource(paths: ["file.txt"])), + destination: "/app/file.txt", + fileMetadata: FileMetadata( + ownership: Ownership(user: .numeric(id: 1000), group: .numeric(id: 1000)), + permissions: .mode(0o644) + ) + ) + #expect(fsOp.action == .copy) + #expect(fsOp.destination == "/app/file.txt") + + // Test MetadataOperation + let metaOp = MetadataOperation( + action: .setLabel(key: "version", value: "1.0.0") + ) + if case .setLabel(let key, let value) = metaOp.action { + #expect(key == "version") + #expect(value == "1.0.0") + } else { + Issue.record("Wrong metadata action type") + } + } + + @Test func serialization() throws { + // Create a graph using the builder + guard let imageRef = ImageReference(parsing: "ubuntu:22.04") else { + Issue.record("Failed to parse image reference") + return + } + + let originalGraph = try GraphBuilder.singleStage( + from: imageRef, + platform: .linuxAMD64 + ) { builder in + try builder + .run("apt-get update && apt-get install -y curl") + .workdir("/app") + .copyFromContext(paths: ["package.json", "src/"], to: "/app/") + .run("npm install") + .env("NODE_ENV", "production") + .expose(3000) + .cmd(Command.exec(["node", "src/index.js"])) + } + + // Serialize to JSON + let coder = JSONIRCoder(prettyPrint: true) + let data = try coder.encode(originalGraph) + + // Deserialize + let decodedGraph = try coder.decode(data) + + // Compare + #expect(originalGraph.stages.count == decodedGraph.stages.count) + #expect(originalGraph.buildArgs == decodedGraph.buildArgs) + #expect(originalGraph.targetPlatforms == decodedGraph.targetPlatforms) + } + + @Test func validation() throws { + // Create a graph with issues + guard let ubuntuRef = ImageReference(parsing: "ubuntu") else { + Issue.record("Failed to parse image reference") + return + } + + let builder = GraphBuilder() + try builder + .stage(from: ubuntuRef) + .run("apt-get update") // Warning: update without install + .copyFromStage(.named("nonexistent"), paths: ["/file"], to: "/") // Error: stage doesn't exist + + // GraphBuilder should throw validation errors during build + #expect(throws: ValidationError.self) { + try builder.build() + } + } + + @Test func semanticAnalysis() throws { + // Create a Python build graph inline + let graph = try BuildGraph( + stages: [ + // Dependencies stage + BuildStage( + name: "dependencies", + base: ImageOperation( + source: .registry(ImageReference(parsing: "python:3.11-slim")!), + platform: .linuxAMD64 + ), + nodes: [ + BuildNode( + operation: MetadataOperation( + action: .setWorkdir(path: "/app") + ) + ), + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .context(ContextSource(paths: ["requirements.txt"])), + destination: "/app/" + ) + ), + BuildNode( + operation: ExecOperation( + command: .shell("pip install --user --no-cache-dir -r requirements.txt") + ) + ), + ] + ), + + // Application stage + BuildStage( + name: "app", + base: ImageOperation( + source: .registry(ImageReference(parsing: "python:3.11-slim")!), + platform: .linuxAMD64 + ), + nodes: [ + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .stage(.named("dependencies"), paths: ["/root/.local"]), + destination: "/root/.local" + ) + ), + BuildNode( + operation: MetadataOperation( + action: .setUser(user: .uidGid(uid: 1000, gid: 1000)) + ) + ), + BuildNode( + operation: MetadataOperation( + action: .setCmd(command: .exec(["python", "main.py"])) + ) + ), + ] + ), + ], + targetPlatforms: [.linuxAMD64, .linuxARM64] + ) + + let analyzer = SemanticAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + let analyzedGraph = try analyzer.analyze(graph, context: context) + + // SemanticAnalyzer should return the graph unchanged + #expect(analyzedGraph.stages.count == graph.stages.count) + #expect(analyzedGraph.buildArgs.count == graph.buildArgs.count) + } + + @Test func graphTraversal() throws { + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .scratch), + nodes: [ + BuildNode(id: UUID(), operation: MetadataOperation(action: .setWorkdir(path: "/app")), dependencies: []), + BuildNode(id: UUID(), operation: ExecOperation(command: .shell("echo test")), dependencies: []), + BuildNode(id: UUID(), operation: MetadataOperation(action: .setUser(user: .uid(1000))), dependencies: []), + ] + ) + + // Test topological sort + let sorted = try GraphTraversal.topologicalSort(stage) + #expect(sorted.count == stage.nodes.count) + + // Test finding roots + let roots = GraphTraversal.findRoots(in: stage) + #expect(roots.count == 3) // All nodes are roots in this case + + // Test finding leaves + let leaves = GraphTraversal.findLeaves(in: stage) + #expect(leaves.count == 3) // All nodes are leaves in this case + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildIRTests/DependencyAnalysisTests.swift b/Tests/NativeBuilderTests/ContainerBuildIRTests/DependencyAnalysisTests.swift new file mode 100644 index 00000000..4cf57eae --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildIRTests/DependencyAnalysisTests.swift @@ -0,0 +1,733 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation +import Testing + +@testable import ContainerBuildIR +@testable import ContainerBuildReporting + +struct DependencyAnalysisTests { + + // MARK: - DependencyAnalyzer Tests + + @Test func sequentialDependenciesInStage() throws { + // Create a stage with multiple operations that should depend on each other + let baseOp = ImageOperation(source: .registry(ImageReference(parsing: "alpine")!)) + let stage = BuildStage( + name: "test", + base: baseOp, + nodes: [ + BuildNode( + operation: ExecOperation(command: .shell("echo 'step 1'")), + dependencies: [] + ), + BuildNode( + operation: ExecOperation(command: .shell("echo 'step 2'")), + dependencies: [] + ), + BuildNode( + operation: ExecOperation(command: .shell("echo 'step 3'")), + dependencies: [] + ), + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let analyzer = DependencyAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let analyzedGraph = try analyzer.analyze(graph, context: context) + let analyzedStage = analyzedGraph.stages[0] + + // First node should have no dependencies + #expect(analyzedStage.nodes[0].dependencies.isEmpty) + + // Second node should depend on first + #expect(analyzedStage.nodes[1].dependencies.contains(analyzedStage.nodes[0].id)) + + // Third node should depend on second + #expect(analyzedStage.nodes[2].dependencies.contains(analyzedStage.nodes[1].id)) + + // Verify chain: node[0] -> node[1] -> node[2] + #expect(analyzedStage.nodes[1].dependencies.count == 1) + #expect(analyzedStage.nodes[2].dependencies.count == 1) + } + + @Test func crossStageDependenciesWithCopyFrom() throws { + guard let alpineRef = ImageReference(parsing: "alpine"), + let ubuntuRef = ImageReference(parsing: "ubuntu") + else { + Issue.record("Failed to parse image references") + return + } + + // Create multi-stage build with COPY --from dependencies + let buildStage = BuildStage( + name: "builder", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: ExecOperation(command: .shell("echo 'building...'")), + dependencies: [] + ), + BuildNode( + operation: ExecOperation(command: .shell("echo 'app built' > /app/binary")), + dependencies: [] + ), + ] + ) + + let runtimeStage = BuildStage( + name: "runtime", + base: ImageOperation(source: .registry(ubuntuRef)), + nodes: [ + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .stage(.named("builder"), paths: ["/app/binary"]), + destination: "/usr/local/bin/app" + ), + dependencies: [] + ), + BuildNode( + operation: MetadataOperation(action: .setEntrypoint(command: .exec(["/usr/local/bin/app"]))), + dependencies: [] + ), + ] + ) + + let graph = try BuildGraph(stages: [buildStage, runtimeStage]) + let analyzer = DependencyAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let analyzedGraph = try analyzer.analyze(graph, context: context) + + // Verify cross-stage dependency was established + let analyzedRuntime = analyzedGraph.stages[1] + let copyNode = analyzedRuntime.nodes[0] + let lastBuildNode = analyzedGraph.stages[0].nodes.last! + + #expect( + copyNode.dependencies.contains(lastBuildNode.id), + "COPY --from should depend on last operation in source stage") + + // Verify intra-stage dependency in runtime stage + let entrypointNode = analyzedRuntime.nodes[1] + #expect( + entrypointNode.dependencies.contains(copyNode.id), + "Entrypoint should depend on copy operation") + } + + @Test func stageReferenceResolution() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let stage1 = BuildStage( + name: "stage1", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: ExecOperation(command: .shell("echo 'stage1' > /file1")), + dependencies: [] + ) + ] + ) + + let stage2 = BuildStage( + name: "stage2", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: ExecOperation(command: .shell("echo 'stage2' > /file2")), + dependencies: [] + ) + ] + ) + + let finalStage = BuildStage( + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + // Test named reference + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .stage(.named("stage1"), paths: ["/file1"]), + destination: "/final/file1" + ), + dependencies: [] + ), + // Test index reference + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .stage(.index(1), paths: ["/file2"]), + destination: "/final/file2" + ), + dependencies: [] + ), + // Test previous reference + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .stage(.previous, paths: ["/file2"]), + destination: "/final/file2-prev" + ), + dependencies: [] + ), + ] + ) + + let graph = try BuildGraph(stages: [stage1, stage2, finalStage]) + let analyzer = DependencyAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let analyzedGraph = try analyzer.analyze(graph, context: context) + let analyzedFinal = analyzedGraph.stages[2] + + // Verify named reference dependency + let namedCopyNode = analyzedFinal.nodes[0] + let stage1LastNode = analyzedGraph.stages[0].nodes.last! + #expect(namedCopyNode.dependencies.contains(stage1LastNode.id)) + + // Verify index reference dependency + let indexCopyNode = analyzedFinal.nodes[1] + let stage2LastNode = analyzedGraph.stages[1].nodes.last! + #expect(indexCopyNode.dependencies.contains(stage2LastNode.id)) + + // Verify previous reference dependency + let previousCopyNode = analyzedFinal.nodes[2] + #expect(previousCopyNode.dependencies.contains(stage2LastNode.id)) + } + + @Test func preserveExistingDependencies() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + // Create nodes with existing dependencies + let node1 = BuildNode( + operation: ExecOperation(command: .shell("echo 'node1'")), + dependencies: [] + ) + + let node2 = BuildNode( + operation: ExecOperation(command: .shell("echo 'node2'")), + dependencies: [] + ) + + let node3 = BuildNode( + operation: ExecOperation(command: .shell("echo 'node3'")), + dependencies: Set([node1.id]) // Explicitly depends on node1, not node2 + ) + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [node1, node2, node3] + ) + + let graph = try BuildGraph(stages: [stage]) + let analyzer = DependencyAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + let analyzedGraph = try analyzer.analyze(graph, context: context) + let analyzedStage = analyzedGraph.stages[0] + + // Node1 should have no dependencies + #expect(analyzedStage.nodes[0].dependencies.isEmpty) + + // Node2 should depend on node1 (sequential) + #expect(analyzedStage.nodes[1].dependencies.contains(analyzedStage.nodes[0].id)) + + // Node3 should preserve its explicit dependency on node1 + // and NOT have sequential dependency on node2 + let node3Analyzed = analyzedStage.nodes[2] + #expect(node3Analyzed.dependencies.contains(node1.id)) + #expect(!node3Analyzed.dependencies.contains(node2.id)) + #expect(node3Analyzed.dependencies.count == 1) + } + + // MARK: - GraphTraversal Tests + + @Test func topologicalSort() throws { + // Create a stage with dependencies: A -> B -> C, A -> D -> C + let nodeA = BuildNode( + operation: ExecOperation(command: .shell("echo 'A'")), + dependencies: [] + ) + let nodeD = BuildNode( + operation: ExecOperation(command: .shell("echo 'D'")), + dependencies: Set([nodeA.id]) + ) + let nodeB = BuildNode( + operation: ExecOperation(command: .shell("echo 'B'")), + dependencies: Set([nodeA.id]) + ) + let nodeC = BuildNode( + operation: ExecOperation(command: .shell("echo 'C'")), + dependencies: Set([nodeB.id, nodeD.id]) + ) + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .scratch), + nodes: [nodeC, nodeB, nodeA, nodeD] // Intentionally out of order + ) + + let sorted = try GraphTraversal.topologicalSort(stage) + + // Find positions in sorted array + let posA = sorted.firstIndex { $0.id == nodeA.id }! + let posB = sorted.firstIndex { $0.id == nodeB.id }! + let posC = sorted.firstIndex { $0.id == nodeC.id }! + let posD = sorted.firstIndex { $0.id == nodeD.id }! + + // Verify ordering constraints + #expect(posA < posB, "A should come before B") + #expect(posA < posD, "A should come before D") + #expect(posB < posC, "B should come before C") + #expect(posD < posC, "D should come before C") + + #expect(sorted.count == 4, "All nodes should be included") + } + + @Test func cycleDetection() throws { + // Create a cycle: A -> B -> C -> A + let nodeA = BuildNode( + operation: ExecOperation(command: .shell("echo 'A'")), + dependencies: [] + ) + let nodeB = BuildNode( + operation: ExecOperation(command: .shell("echo 'B'")), + dependencies: Set([nodeA.id]) + ) + let nodeC = BuildNode( + operation: ExecOperation(command: .shell("echo 'C'")), + dependencies: Set([nodeB.id]) + ) + + // Create cycle by making A depend on C + let nodeACyclic = BuildNode( + id: nodeA.id, + operation: nodeA.operation, + dependencies: Set([nodeC.id]) + ) + + let stage = BuildStage( + name: "cyclic", + base: ImageOperation(source: .scratch), + nodes: [nodeACyclic, nodeB, nodeC] + ) + + #expect(throws: BuildGraphError.self) { + try GraphTraversal.topologicalSort(stage) + } + } + + @Test func findDependentsAndDependencies() throws { + // Create dependency chain: A -> B -> C, A -> D + let nodeA = BuildNode( + operation: ExecOperation(command: .shell("echo 'A'")), + dependencies: [] + ) + let nodeB = BuildNode( + operation: ExecOperation(command: .shell("echo 'B'")), + dependencies: Set([nodeA.id]) + ) + let nodeC = BuildNode( + operation: ExecOperation(command: .shell("echo 'C'")), + dependencies: Set([nodeB.id]) + ) + let nodeD = BuildNode( + operation: ExecOperation(command: .shell("echo 'D'")), + dependencies: Set([nodeA.id]) + ) + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .scratch), + nodes: [nodeA, nodeB, nodeC, nodeD] + ) + + // Test finding dependents + let aDependents = GraphTraversal.findDependents(of: nodeA.id, in: stage) + #expect(aDependents.contains(nodeB.id)) + #expect(aDependents.contains(nodeC.id)) // Transitive + #expect(aDependents.contains(nodeD.id)) + #expect(aDependents.count == 3) + + let bDependents = GraphTraversal.findDependents(of: nodeB.id, in: stage) + #expect(bDependents.contains(nodeC.id)) + #expect(!bDependents.contains(nodeA.id)) + #expect(bDependents.count == 1) + + // Test finding dependencies + let cDependencies = GraphTraversal.findDependencies(of: nodeC.id, in: stage) + #expect(cDependencies.contains(nodeB.id)) + #expect(cDependencies.contains(nodeA.id)) // Transitive + #expect(!cDependencies.contains(nodeD.id)) + #expect(cDependencies.count == 2) + + let aDependencies = GraphTraversal.findDependencies(of: nodeA.id, in: stage) + #expect(aDependencies.isEmpty, "Root node should have no dependencies") + } + + @Test func findRootsAndLeaves() throws { + // Create graph: A -> B -> C, D -> E + let nodeA = BuildNode( + operation: ExecOperation(command: .shell("echo 'A'")), + dependencies: [] + ) + let nodeB = BuildNode( + operation: ExecOperation(command: .shell("echo 'B'")), + dependencies: Set([nodeA.id]) + ) + let nodeC = BuildNode( + operation: ExecOperation(command: .shell("echo 'C'")), + dependencies: Set([nodeB.id]) + ) + let nodeD = BuildNode( + operation: ExecOperation(command: .shell("echo 'D'")), + dependencies: [] + ) + let nodeE = BuildNode( + operation: ExecOperation(command: .shell("echo 'E'")), + dependencies: Set([nodeD.id]) + ) + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .scratch), + nodes: [nodeA, nodeB, nodeC, nodeD, nodeE] + ) + + // Test finding roots + let roots = GraphTraversal.findRoots(in: stage) + let rootIds = Set(roots.map { $0.id }) + #expect(rootIds.contains(nodeA.id)) + #expect(rootIds.contains(nodeD.id)) + #expect(rootIds.count == 2) + + // Test finding leaves + let leaves = GraphTraversal.findLeaves(in: stage) + let leafIds = Set(leaves.map { $0.id }) + #expect(leafIds.contains(nodeC.id)) + #expect(leafIds.contains(nodeE.id)) + #expect(leafIds.count == 2) + } + + @Test func criticalPath() throws { + // Create a diamond dependency: A -> B -> D, A -> C -> D + let nodeA = BuildNode( + operation: ExecOperation(command: .shell("echo 'A'")), + dependencies: [] + ) + let nodeB = BuildNode( + operation: ExecOperation(command: .shell("echo 'B'")), + dependencies: Set([nodeA.id]) + ) + let nodeC = BuildNode( + operation: ExecOperation(command: .shell("echo 'C'")), + dependencies: Set([nodeA.id]) + ) + let nodeD = BuildNode( + operation: ExecOperation(command: .shell("echo 'D'")), + dependencies: Set([nodeB.id, nodeC.id]) + ) + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .scratch), + nodes: [nodeA, nodeB, nodeC, nodeD] + ) + + let criticalPath = GraphTraversal.criticalPath(in: stage) + + #expect(criticalPath.count >= 3, "Critical path should have at least 3 nodes") + + // Verify path contains A and D (start and end) + let pathIds = Set(criticalPath.map { $0.id }) + #expect(pathIds.contains(nodeA.id)) + #expect(pathIds.contains(nodeD.id)) + } + + @Test func depthFirstTraversal() throws { + // Create dependency chain: A -> B -> C + let nodeA = BuildNode( + operation: ExecOperation(command: .shell("echo 'A'")), + dependencies: [] + ) + let nodeB = BuildNode( + operation: ExecOperation(command: .shell("echo 'B'")), + dependencies: Set([nodeA.id]) + ) + let nodeC = BuildNode( + operation: ExecOperation(command: .shell("echo 'C'")), + dependencies: Set([nodeB.id]) + ) + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .scratch), + nodes: [nodeC, nodeB, nodeA] // Out of order + ) + + var visitOrder: [UUID] = [] + + try GraphTraversal.depthFirst(stage: stage) { node in + visitOrder.append(node.id) + } + + #expect(visitOrder.count == 3) + + // Find positions in visit order + let posA = visitOrder.firstIndex(of: nodeA.id)! + let posB = visitOrder.firstIndex(of: nodeB.id)! + let posC = visitOrder.firstIndex(of: nodeC.id)! + + // DFS should visit dependencies before dependents + #expect(posA < posB, "A should be visited before B") + #expect(posB < posC, "B should be visited before C") + } + + // MARK: - Stage Dependencies Tests + + @Test func findStageDependencies() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let stage1 = BuildStage( + name: "build", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: ExecOperation(command: .shell("echo 'building...'")), + dependencies: [] + ) + ] + ) + + let stage2 = BuildStage( + name: "test", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .stage(.named("build"), paths: ["/app"]), + destination: "/test/app" + ), + dependencies: [] + ) + ] + ) + + let stage3 = BuildStage( + name: "final", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .stage(.index(0), paths: ["/app"]), + destination: "/final/app" + ), + dependencies: [] + ), + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .stage(.previous, paths: ["/test"]), + destination: "/final/test" + ), + dependencies: [] + ), + ] + ) + + let graph = try BuildGraph(stages: [stage1, stage2, stage3]) + + // Test stage 2 dependencies + let stage2Deps = GraphTraversal.findStageDependencies(of: stage2, in: graph) + #expect(stage2Deps.contains("build")) + #expect(stage2Deps.count == 1) + + // Test stage 3 dependencies + let stage3Deps = GraphTraversal.findStageDependencies(of: stage3, in: graph) + #expect(stage3Deps.contains("build")) // From index(0) + #expect(stage3Deps.contains("test")) // From .previous + #expect(stage3Deps.count == 2) + + // Test stage 1 dependencies (should be empty) + let stage1Deps = GraphTraversal.findStageDependencies(of: stage1, in: graph) + #expect(stage1Deps.isEmpty) + } + + // MARK: - Build Graph Analysis Tests + + @Test func buildGraphAnalysis() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let graph = try GraphBuilder.multiStage { builder in + try builder + .stage(name: "build", from: alpineRef) + .run("apk add --no-cache build-tools") + .workdir("/src") + .copyFromContext(paths: ["src/"], to: "./") + .run("make build") + + try builder + .stage(name: "test", from: alpineRef) + .copyFromStage(.named("build"), paths: ["/src/app"], to: "/test/") + .run("./test/app --test") + + try builder + .stage(from: alpineRef) + .copyFromStage(.named("build"), paths: ["/src/app"], to: "/usr/local/bin/") + .entrypoint(.exec(["/usr/local/bin/app"])) + } + + let analysis = graph.analyze() + + #expect(analysis.stageCount == 3) + #expect(analysis.operationCount > 0) + + // Verify operation types are counted + #expect(analysis.operationsByType[OperationKind.exec] != nil) + #expect(analysis.operationsByType[OperationKind.filesystem] != nil) + #expect(analysis.operationsByType[OperationKind.metadata] != nil) + + // Verify stage dependencies + #expect(analysis.stageDependencies["test"]?.contains("build") == true) + #expect(analysis.stageDependencies.count >= 1) + + #expect(analysis.maxDepth > 0) + #expect(analysis.criticalPathLength > 0) + } + + // MARK: - Error Handling Tests + + @Test func invalidStageReferenceHandling() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + // Create stage with invalid reference + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .stage(.named("nonexistent"), paths: ["/file"]), + destination: "/test/file" + ), + dependencies: [] + ) + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let analyzer = DependencyAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + + // Should not throw - dependency analyzer handles missing references gracefully + let analyzedGraph = try analyzer.analyze(graph, context: context) + + // The copy operation should not have gained any cross-stage dependencies + let copyNode = analyzedGraph.stages[0].nodes[0] + #expect( + copyNode.dependencies.isEmpty, + "Copy from nonexistent stage should not create dependencies") + } + + @Test func complexDependencyChain() throws { + // Test a complex multi-stage build with intricate dependencies + guard let nodeRef = ImageReference(parsing: "node:18"), + let nginxRef = ImageReference(parsing: "nginx:alpine") + else { + Issue.record("Failed to parse image references") + return + } + + let graph = try GraphBuilder.multiStage { builder in + // Dependencies stage + try builder + .stage(name: "deps", from: nodeRef) + .workdir("/app") + .copyFromContext(paths: ["package*.json"], to: "./") + .run("npm ci --only=production") + + // Build stage + try builder + .stage(name: "build", from: nodeRef) + .workdir("/app") + .copyFromStage(.named("deps"), paths: ["/app/node_modules"], to: "/app/node_modules") + .copyFromContext(paths: ["src/", "tsconfig.json"], to: "./") + .run("npm run build") + + // Assets stage + try builder + .stage(name: "assets", from: nodeRef) + .workdir("/app") + .copyFromStage(.named("build"), paths: ["/app/dist"], to: "/app/dist") + .run("npm run optimize-assets") + + // Final stage + try builder + .stage(from: nginxRef) + .copyFromStage(.named("assets"), paths: ["/app/dist"], to: "/usr/share/nginx/html") + .copyFromContext(paths: ["nginx.conf"], to: "/etc/nginx/nginx.conf") + .expose(80) + } + + let analyzer = DependencyAnalyzer() + let context = AnalysisContext(reporter: nil, sourceMap: nil) + let analyzedGraph = try analyzer.analyze(graph, context: context) + + // Verify complex dependency chain + let finalStage = analyzedGraph.stages[3] + let copyFromAssetsNode = finalStage.nodes[0] + let assetsStageLastNode = analyzedGraph.stages[2].nodes.last! + + #expect(copyFromAssetsNode.dependencies.contains(assetsStageLastNode.id)) + + // Verify transitive dependencies exist through the chain + let assetsStage = analyzedGraph.stages[2] + let copyFromBuildNode = assetsStage.nodes[1] // Node 1 is the copyFromStage operation + let buildStageLastNode = analyzedGraph.stages[1].nodes.last! + + #expect(copyFromBuildNode.dependencies.contains(buildStageLastNode.id)) + + // Verify build stage dependencies + let buildStage = analyzedGraph.stages[1] + let copyFromDepsNode = buildStage.nodes[1] // Node 1 is the copyFromStage operation + let depsStageLastNode = analyzedGraph.stages[0].nodes.last! + + #expect(copyFromDepsNode.dependencies.contains(depsStageLastNode.id)) + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildIRTests/DigestAndPlatformTests.swift b/Tests/NativeBuilderTests/ContainerBuildIRTests/DigestAndPlatformTests.swift new file mode 100644 index 00000000..577894d4 --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildIRTests/DigestAndPlatformTests.swift @@ -0,0 +1,323 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOCI +import Foundation +import Testing + +@testable import ContainerBuildIR + +struct DigestAndPlatformTests { + + // MARK: - Digest Algorithm Tests + + @Test func digestSHA256Creation() throws { + let data = "Hello, World!".data(using: .utf8)! + let digest = try Digest.compute(data, using: .sha256) + + #expect(digest.algorithm == .sha256) + #expect(digest.bytes.count == 32) // SHA256 produces 32 bytes + + // Verify deterministic computation + let digest2 = try Digest.compute(data, using: .sha256) + #expect(digest == digest2) + + // Verify string format + let stringValue = digest.stringValue + #expect(stringValue.hasPrefix("sha256:")) + #expect(stringValue.count == "sha256:".count + 64) // 32 bytes = 64 hex chars + } + + @Test func digestSHA384Creation() throws { + let data = "Test data for SHA384".data(using: .utf8)! + let digest = try Digest.compute(data, using: .sha384) + + #expect(digest.algorithm == .sha384) + #expect(digest.bytes.count == 48) // SHA384 produces 48 bytes + + // Verify string format + let stringValue = digest.stringValue + #expect(stringValue.hasPrefix("sha384:")) + #expect(stringValue.count == "sha384:".count + 96) // 48 bytes = 96 hex chars + } + + @Test func digestSHA512Creation() throws { + let data = "Test data for SHA512".data(using: .utf8)! + let digest = try Digest.compute(data, using: .sha512) + + #expect(digest.algorithm == .sha512) + #expect(digest.bytes.count == 64) // SHA512 produces 64 bytes + + // Verify string format + let stringValue = digest.stringValue + #expect(stringValue.hasPrefix("sha512:")) + #expect(stringValue.count == "sha512:".count + 128) // 64 bytes = 128 hex chars + } + + @Test func digestFromValidBytes() throws { + // Create SHA256 digest from valid bytes + let validBytes = Data(repeating: 0xAB, count: 32) + let digest = try Digest(algorithm: .sha256, bytes: validBytes) + + #expect(digest.algorithm == .sha256) + #expect(digest.bytes == validBytes) + + let expectedString = "sha256:" + String(repeating: "ab", count: 32) + #expect(digest.stringValue == expectedString) + } + + @Test func digestFromInvalidBytes() throws { + // Test wrong length for SHA256 + let wrongLengthBytes = Data(repeating: 0xFF, count: 16) // Should be 32 + #expect(throws: DigestError.self) { + try Digest(algorithm: .sha256, bytes: wrongLengthBytes) + } + + // Test wrong length for SHA384 + let wrongLengthBytes384 = Data(repeating: 0xFF, count: 32) // Should be 48 + #expect(throws: DigestError.self) { + try Digest(algorithm: .sha384, bytes: wrongLengthBytes384) + } + + // Test wrong length for SHA512 + let wrongLengthBytes512 = Data(repeating: 0xFF, count: 32) // Should be 64 + #expect(throws: DigestError.self) { + try Digest(algorithm: .sha512, bytes: wrongLengthBytes512) + } + } + + // MARK: - Digest Parsing Tests + + @Test func digestParsingValidFormats() throws { + let testCases = [ + ("sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", Digest.Algorithm.sha256), + ("sha384:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", Digest.Algorithm.sha384), + ("sha512:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", Digest.Algorithm.sha512), + ] + + for (digestString, expectedAlgorithm) in testCases { + let digest = try Digest(parsing: digestString) + #expect(digest.algorithm == expectedAlgorithm) + #expect(digest.stringValue == digestString) + } + } + + @Test func digestParsingInvalidFormats() throws { + let invalidFormats = [ + "no-colon-separator", + "sha256", + "sha256:", + "sha256:invalid-hex", + "sha256:zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", // Invalid hex chars + "unknown:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", + "sha256:short", // Too short + "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789extra", // Too long + ] + + for invalidFormat in invalidFormats { + #expect(throws: DigestError.self) { + try Digest(parsing: invalidFormat) + } + } + } + + @Test func digestParsingMixedCase() throws { + // Test that mixed case hex is handled correctly + let upperCaseDigest = "sha256:ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789" + let lowerCaseDigest = "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" + let mixedCaseDigest = "sha256:AbCdEf0123456789aBcDeF0123456789AbCdEf0123456789aBcDeF0123456789" + + let upperDigest = try Digest(parsing: upperCaseDigest) + let lowerDigest = try Digest(parsing: lowerCaseDigest) + let mixedDigest = try Digest(parsing: mixedCaseDigest) + + // All should parse to the same bytes + #expect(upperDigest.bytes == lowerDigest.bytes) + #expect(lowerDigest.bytes == mixedDigest.bytes) + + // String output should be lowercase + #expect(upperDigest.stringValue == lowerCaseDigest) + #expect(mixedDigest.stringValue == lowerCaseDigest) + } + + // MARK: - Digest Content Hashing Tests + + @Test func digestContentHashing() throws { + // Test that different content produces different digests + let content1 = "First piece of content".data(using: .utf8)! + let content2 = "Second piece of content".data(using: .utf8)! + + let digest1 = try Digest.compute(content1) + let digest2 = try Digest.compute(content2) + + #expect(digest1 != digest2) + #expect(digest1.stringValue != digest2.stringValue) + + // Test that same content produces same digest + let digest1Copy = try Digest.compute(content1) + #expect(digest1 == digest1Copy) + } + + @Test func digestEmptyContent() throws { + let emptyData = Data() + let digest = try Digest.compute(emptyData) + + #expect(digest.algorithm == .sha256) // Default algorithm + #expect(digest.bytes.count == 32) + + // Known SHA256 of empty string + let expectedEmptyDigest = try Digest(parsing: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + #expect(digest == expectedEmptyDigest) + } + + @Test func digestLargeContent() throws { + // Test with larger content (1MB) + let largeContent = Data(repeating: 0x42, count: 1024 * 1024) + let digest = try Digest.compute(largeContent) + + #expect(digest.algorithm == .sha256) + #expect(digest.bytes.count == 32) + + // Verify deterministic + let digest2 = try Digest.compute(largeContent) + #expect(digest == digest2) + } + + // MARK: - Digest Codable Tests + + @Test func digestCodable() throws { + let originalDigest = try Digest(parsing: "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789") + + // Encode to JSON + let encoder = JSONEncoder() + let data = try encoder.encode(originalDigest) + + // Decode from JSON + let decoder = JSONDecoder() + let decodedDigest = try decoder.decode(Digest.self, from: data) + + #expect(decodedDigest == originalDigest) + #expect(decodedDigest.stringValue == originalDigest.stringValue) + } + + // MARK: - Integration Tests + + @Test func digestAndPlatformIntegration() throws { + // Test that digests and platforms work together in realistic scenarios + let platform = Platform.linuxAMD64 + let operationData = "RUN apt-get update && apt-get install -y curl".data(using: .utf8)! + let operationDigest = try Digest.compute(operationData) + + // Simulate cache key generation (simplified) + let cacheKeyData = "\(operationDigest.stringValue):\(platform.description)".data(using: .utf8)! + let cacheDigest = try Digest.compute(cacheKeyData) + + #expect(cacheDigest.algorithm == .sha256) + #expect(cacheDigest.bytes.count == 32) + + // Different platform should produce different cache key + let differentPlatform = Platform.linuxARM64 + let differentCacheKeyData = "\(operationDigest.stringValue):\(differentPlatform.description)".data(using: .utf8)! + let differentCacheDigest = try Digest.compute(differentCacheKeyData) + + #expect(cacheDigest != differentCacheDigest) + } + + @Test func multiAlgorithmDigestComparison() throws { + let testData = "Container build test data".data(using: .utf8)! + + let sha256Digest = try Digest.compute(testData, using: .sha256) + let sha384Digest = try Digest.compute(testData, using: .sha384) + let sha512Digest = try Digest.compute(testData, using: .sha512) + + // All should be different (different algorithms) + #expect(sha256Digest != sha384Digest) + #expect(sha384Digest != sha512Digest) + #expect(sha256Digest != sha512Digest) + + // Verify byte lengths + #expect(sha256Digest.bytes.count == 32) + #expect(sha384Digest.bytes.count == 48) + #expect(sha512Digest.bytes.count == 64) + + // Verify string prefixes + #expect(sha256Digest.stringValue.hasPrefix("sha256:")) + #expect(sha384Digest.stringValue.hasPrefix("sha384:")) + #expect(sha512Digest.stringValue.hasPrefix("sha512:")) + } + + // MARK: - Error Message Tests + + @Test func digestErrorMessages() throws { + do { + try Digest(algorithm: .sha256, bytes: Data(count: 16)) + Issue.record("Should have thrown an error") + } catch let error as DigestError { + switch error { + case .invalidLength(let expected, let actual): + #expect(expected == 32) + #expect(actual == 16) + #expect(error.errorDescription?.contains("expected 32") == true) + #expect(error.errorDescription?.contains("got 16") == true) + default: + Issue.record("Wrong error type: \(error)") + } + } + + do { + try Digest(parsing: "invalid:format") + Issue.record("Should have thrown an error") + } catch let error as DigestError { + switch error { + case .unsupportedAlgorithm(let algo): + #expect(algo == "invalid") + #expect(error.errorDescription?.contains("Unsupported digest algorithm") == true) + default: + Issue.record("Wrong error type: \(error)") + } + } + + do { + try Digest(parsing: "sha256:invalid-hex") + Issue.record("Should have thrown an error") + } catch let error as DigestError { + switch error { + case .invalidHex(let hex): + #expect(hex == "invalid-hex") + #expect(error.errorDescription?.contains("Invalid hex") == true) + default: + Issue.record("Wrong error type: \(error)") + } + } + } + + // MARK: - Performance Tests + + @Test func digestPerformance() throws { + let testSizes = [1024, 10240, 102400] // 1KB, 10KB, 100KB + + for size in testSizes { + let data = Data(repeating: 0x42, count: size) + + let startTime = Date() + let _ = try Digest.compute(data) + let duration = Date().timeIntervalSince(startTime) + + print("Digest computation for \(size) bytes: \(String(format: "%.3f", duration))s") + #expect(duration < 0.1, "Digest computation should be fast for \(size) bytes") + } + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildIRTests/GraphBuilderTests.swift b/Tests/NativeBuilderTests/ContainerBuildIRTests/GraphBuilderTests.swift new file mode 100644 index 00000000..753f3410 --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildIRTests/GraphBuilderTests.swift @@ -0,0 +1,558 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOCI +import Foundation +import Testing + +@testable import ContainerBuildIR +@testable import ContainerBuildReporting + +struct GraphBuilderTests { + + // MARK: - Complex Multi-Stage Scenarios + + @Test func complexMultiStageNodeJSBuild() throws { + guard let nodeRef = ImageReference(parsing: "node:18-alpine"), + let nginxRef = ImageReference(parsing: "nginx:alpine") + else { + Issue.record("Failed to parse image references") + return + } + + let graph = try GraphBuilder.multiStage { builder in + // Dependencies stage - install and cache node_modules + try builder + .stage(name: "deps", from: nodeRef) + .workdir("/app") + .copyFromContext(paths: ["package*.json"], to: "./") + .run("npm ci --only=production && npm cache clean --force") + + // Build stage - compile TypeScript and build assets + try builder + .stage(name: "build", from: nodeRef) + .workdir("/app") + .copyFromStage(.named("deps"), paths: ["/app/node_modules"], to: "/app/node_modules") + .copyFromContext(paths: ["tsconfig.json", "webpack.config.js", "src/"], to: "./") + .run("npm run build") + + // Runtime stage - serve with nginx + try builder + .stage(name: "runtime", from: nginxRef) + .copyFromStage(.named("build"), paths: ["/app/dist"], to: "/usr/share/nginx/html") + .copyFromContext(paths: ["nginx.conf"], to: "/etc/nginx/nginx.conf") + .expose(80) + .cmd(.exec(["nginx", "-g", "daemon off;"])) + } + + #expect(graph.stages.count == 3) + #expect(graph.stages[0].name == "deps") + #expect(graph.stages[1].name == "build") + #expect(graph.stages[2].name == "runtime") + + // Verify stage dependencies + let buildDeps = graph.stages[1].stageDependencies() + #expect(buildDeps.contains(.named("deps"))) + + let runtimeDeps = graph.stages[2].stageDependencies() + #expect(runtimeDeps.contains(.named("build"))) + + // Validate the complex graph + let validator = StandardValidator() + let result = validator.validate(graph) + #expect(result.isValid, "Complex multi-stage build should be valid: \(result.errors)") + } + + @Test func fourStageGoMicroserviceBuild() throws { + guard let goRef = ImageReference(parsing: "golang:1.21-alpine"), + let scratchRef = ImageReference(parsing: "scratch") + else { + Issue.record("Failed to parse image references") + return + } + + let graph = try GraphBuilder.multiStage { builder in + // Base tools stage + try builder + .stage(name: "tools", from: goRef) + .run("apk add --no-cache git ca-certificates") + .workdir("/tools") + .run("go install github.com/swaggo/swag/cmd/swag@latest") + + // Dependency stage + try builder + .stage(name: "deps", from: goRef) + .workdir("/src") + .copyFromContext(paths: ["go.mod", "go.sum"], to: "./") + .run("go mod download") + + // Build stage + try builder + .stage(name: "build", from: goRef) + .copyFromStage(.named("tools"), paths: ["/go/bin/swag"], to: "/usr/local/bin/") + .workdir("/src") + .copyFromStage(.named("deps"), paths: ["/go/pkg"], to: "/go/pkg") + .copyFromContext(paths: [".", "!**/*_test.go"], to: "./") + .run("swag init -g cmd/server/main.go") + .run("CGO_ENABLED=0 GOOS=linux go build -ldflags='-w -s' -o /app cmd/server/main.go") + + // Runtime stage + try builder + .stage(from: scratchRef) + .copyFromStage(.named("tools"), paths: ["/etc/ssl/certs/ca-certificates.crt"], to: "/etc/ssl/certs/") + .copyFromStage(.named("build"), paths: ["/app"], to: "/app") + .user(.uid(65534)) // nobody user + .expose(8080) + .entrypoint(.exec(["/app"])) + } + + #expect(graph.stages.count == 4) + + // Verify complex dependency chain + let buildStage = graph.stages[2] + let buildDeps = buildStage.stageDependencies() + #expect(buildDeps.contains(.named("tools"))) + #expect(buildDeps.contains(.named("deps"))) + + let runtimeStage = graph.stages[3] + let runtimeDeps = runtimeStage.stageDependencies() + #expect(runtimeDeps.contains(.named("tools"))) + #expect(runtimeDeps.contains(.named("build"))) + } + + // MARK: - Build Arguments and Environment + + @Test func buildArgumentPropagation() throws { + guard let nodeRef = ImageReference(parsing: "node:18") else { + Issue.record("Failed to parse image reference") + return + } + + let graph = try GraphBuilder.multiStage { builder in + try builder + .stage(name: "build", from: nodeRef) + .arg("NODE_ENV", defaultValue: "development") + .arg("BUILD_VERSION", defaultValue: "dev") + .arg("API_URL") + .workdir("/app") + .copyFromContext(paths: ["package.json"], to: "./") + .run("npm install --omit=dev") + .env("NODE_ENV", "production") + .env("BUILD_VERSION", "1.0.0") + .env("API_URL", "https://api.example.com") + .copyFromContext(paths: ["src/"], to: "./src/") + .run("npm run build") + } + + #expect(graph.buildArgs.count == 2) + #expect(graph.buildArgs["NODE_ENV"] == "development") + #expect(graph.buildArgs["BUILD_VERSION"] == "dev") + // API_URL has no default value so it's not included in buildArgs + + // Verify ARG instructions in stage + let stage = graph.stages[0] + let argOps = stage.nodes.compactMap { node in + node.operation as? MetadataOperation + }.filter { meta in + if case .declareArg = meta.action { return true } + return false + } + #expect(argOps.count == 3) + + // Verify ENV instructions reference build args + let envOps = stage.nodes.compactMap { node in + node.operation as? MetadataOperation + }.filter { meta in + if case .setEnv = meta.action { return true } + return false + } + #expect(envOps.count == 3) + } + + // MARK: - Platform-Specific Builds + + @Test func multiPlatformBuild() throws { + guard let baseRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + let graph = try GraphBuilder.singleStage(from: baseRef, platform: Platform.linuxAMD64) { builder in + try builder + .platforms(Platform.linuxAMD64, Platform.linuxARM64) + .run("apk add --no-cache curl") + .workdir("/app") + .copyFromContext(paths: ["app.sh"], to: "/app/") + .run("chmod +x /app/app.sh") + .entrypoint(.exec(["/app/app.sh"])) + } + + #expect(graph.targetPlatforms.count == 2) + #expect(graph.targetPlatforms.contains(Platform.linuxAMD64)) + #expect(graph.targetPlatforms.contains(Platform.linuxARM64)) + + // Verify base image has platform constraints + let stage = graph.stages[0] + let imageOp = stage.base + #expect(imageOp.platform != nil) + } + + @Test func platformSpecificStages() throws { + guard let alpineRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + let graph = try GraphBuilder.multiStage { builder in + // Linux AMD64 optimized stage + try builder + .stage(name: "linux-amd64", from: alpineRef, platform: Platform.linuxAMD64) + .run("apk add --no-cache glibc-compat") + .copyFromContext(paths: ["bin/app-linux-amd64"], to: "/usr/local/bin/app") + + // Linux ARM64 stage + try builder + .stage(name: "linux-arm64", from: alpineRef, platform: Platform.linuxARM64) + .run("apk add --no-cache ca-certificates") + .copyFromContext(paths: ["bin/app-linux-arm64"], to: "/usr/local/bin/app") + + // Final stage that copies from platform-specific stage + try builder + .stage(from: alpineRef) + .copyFromStage(.named("linux-amd64"), paths: ["/usr/local/bin/app"], to: "/usr/local/bin/") + .copyFromStage(.named("linux-arm64"), paths: ["/usr/local/bin/app"], to: "/usr/local/bin/") + .entrypoint(.exec(["/usr/local/bin/app"])) + } + + #expect(graph.stages.count == 3) + + // Verify platform-specific stages + #expect(graph.stages[0].name == "linux-amd64") + #expect(graph.stages[1].name == "linux-arm64") + + let amd64ImageOp = graph.stages[0].base + #expect(amd64ImageOp.platform == Platform.linuxAMD64) + + let arm64ImageOp = graph.stages[1].base + #expect(arm64ImageOp.platform == Platform.linuxARM64) + } + + // MARK: - Error Conditions + + @Test func invalidStageReference() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + #expect(throws: Error.self) { + try GraphBuilder.multiStage { builder in + try builder + .stage(name: "base", from: alpineRef) + .run("echo hello") + + try builder + .stage(name: "app", from: alpineRef) + .copyFromStage(.named("nonexistent"), paths: ["/file"], to: "/") + } + } + } + + @Test func circularStageDependency() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + // This should create a circular dependency through stage references + // GraphBuilder should throw BuildGraphError for cyclic dependencies + #expect(throws: BuildGraphError.self) { + try GraphBuilder.multiStage { builder in + try builder + .stage(name: "stage1", from: alpineRef) + .copyFromStage(.named("stage2"), paths: ["/file1"], to: "/file1") + + try builder + .stage(name: "stage2", from: alpineRef) + .copyFromStage(.named("stage1"), paths: ["/file2"], to: "/file2") + } + } + } + + @Test func validEntrypointAndCmdSequence() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + // CMD after ENTRYPOINT is a valid and common Docker pattern + // ENTRYPOINT defines the executable, CMD provides default arguments + let graph = try GraphBuilder.singleStage(from: alpineRef) { builder in + try builder + .entrypoint(.exec(["/app"])) // Set entrypoint first + .cmd(.shell("echo override")) // CMD after ENTRYPOINT is valid + } + + #expect(graph.stages.count == 1) + #expect(graph.stages[0].nodes.count == 2) + + // Verify the operations exist and are in the correct order + let nodes = graph.stages[0].nodes + + let entrypointNode = nodes.first { node in + if let metaOp = node.operation as? MetadataOperation, + case .setEntrypoint = metaOp.action + { + return true + } + return false + } + #expect(entrypointNode != nil, "Should have ENTRYPOINT operation") + + let cmdNode = nodes.first { node in + if let metaOp = node.operation as? MetadataOperation, + case .setCmd = metaOp.action + { + return true + } + return false + } + #expect(cmdNode != nil, "Should have CMD operation") + + // Validate the graph structure + let validator = StandardValidator() + let result = validator.validate(graph) + #expect(result.isValid, "ENTRYPOINT + CMD sequence should be valid: \(result.errors)") + } + + @Test func emptyStage() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + // Empty stage should be valid but might generate warnings + let graph = try GraphBuilder.singleStage(from: alpineRef) { builder in + // Intentionally empty - just the base image + } + + #expect(graph.stages.count == 1) + #expect(graph.stages[0].nodes.isEmpty, "Empty stage should have no nodes") + + let validator = StandardValidator() + let result = validator.validate(graph) + #expect(result.isValid, "Empty stage should be structurally valid") + } + + // MARK: - Advanced GraphBuilder Features + + @Test func conditionalInstructions() throws { + guard let nodeRef = ImageReference(parsing: "node:18") else { + Issue.record("Failed to parse image reference") + return + } + + let isDevelopment = false + + let graph = try GraphBuilder.singleStage(from: nodeRef) { builder in + try builder + .workdir("/app") + .copyFromContext(paths: ["package.json"], to: "./") + .run("npm install" + (isDevelopment ? "" : " --omit=dev")) + .copyFromContext(paths: ["src/"], to: "./src/") + + if isDevelopment { + try builder + .env("NODE_ENV", "development") + .run("npm run test") + .cmd(.shell("npm run dev")) + } else { + try builder + .env("NODE_ENV", "production") + .run("npm run build") + .cmd(.exec(["node", "dist/index.js"])) + } + } + + // Verify production build path was taken + let envOps = graph.stages[0].nodes.compactMap { node in + node.operation as? MetadataOperation + }.compactMap { meta in + if case .setEnv(let key, let value) = meta.action, + key == "NODE_ENV" + { + return value + } + return nil + } + + #expect(envOps.contains(.literal("production"))) + #expect(!envOps.contains(.literal("development"))) + } + + @Test func stageWithCustomMetadata() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let graph = try GraphBuilder.singleStage(from: alpineRef) { builder in + try builder + .label("version", "1.0.0") + .label("maintainer", "team@example.com") + .label("description", "Sample application") + .workdir("/app") + .user(.named("appuser")) + .expose(8080) + } + + // Verify all metadata operations were added + let metadataOps = graph.stages[0].nodes.compactMap { $0.operation as? MetadataOperation } + + let labelOps = metadataOps.filter { + if case .setLabel = $0.action { return true } + return false + } + #expect(labelOps.count == 3) + + let userOps = metadataOps.filter { + if case .setUser = $0.action { return true } + return false + } + #expect(userOps.count == 1) + + let exposeOps = metadataOps.filter { + if case .expose = $0.action { return true } + return false + } + #expect(exposeOps.count == 1) + } + + @Test func complexMountOperations() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let graph = try GraphBuilder.singleStage(from: alpineRef) { builder in + try builder + .workdir("/app") + .run( + "apk add --no-cache make gcc", + mounts: [ + Mount(type: .cache, target: "/var/cache/apk"), + Mount(type: .secret, target: "/run/secrets/github-token", source: .secret("github-token"), options: MountOptions(mode: 0o400)), + Mount(type: .bind, target: "/app/.cache", source: .local("build-cache")), + ] + ) + .copyFromContext(paths: ["Makefile", "src/"], to: "./") + .run( + "make build", + mounts: [ + Mount(type: .cache, target: "/app/.cache", options: MountOptions(sharing: .shared)), + Mount(type: .tmpfs, target: "/tmp", options: MountOptions(size: 100_000_000)), + ] + ) + } + + // Verify mount operations exist + let execOps = graph.stages[0].nodes.compactMap { $0.operation as? ExecOperation } + + let firstRunOp = execOps.first { op in + op.command.displayString.contains("apk add") + } + #expect(firstRunOp?.mounts.count == 3) + + let secondRunOp = execOps.first { op in + op.command.displayString.contains("make build") + } + #expect(secondRunOp?.mounts.count == 2) + } + + // MARK: - Graph Modification and Builder State + + @Test func builderStatePreservation() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let graph1 = try GraphBuilder.singleStage(from: alpineRef) { stageBuilder in + try stageBuilder + .arg("VERSION", defaultValue: "1.0.0") + .platforms(Platform.linuxAMD64) + .run("echo 'Build 1'") + } + + // Verify first graph has our settings + #expect(graph1.buildArgs["VERSION"] == "1.0.0") + #expect(graph1.targetPlatforms == [Platform.linuxAMD64]) + + // Create second graph with different settings + let graph2 = try GraphBuilder.singleStage(from: alpineRef) { stageBuilder in + try stageBuilder + .arg("VERSION", defaultValue: "2.0.0") + .platforms(Platform.linuxARM64) + .run("echo 'Build 2'") + } + + // Verify second graph has updated settings + #expect(graph2.buildArgs["VERSION"] == "2.0.0") + #expect(graph2.targetPlatforms == [Platform.linuxARM64]) + + // Verify first graph unchanged + #expect(graph1.buildArgs["VERSION"] == "1.0.0") + #expect(graph1.targetPlatforms == [Platform.linuxAMD64]) + } + + @Test func incrementalGraphBuilding() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let graph = try GraphBuilder.multiStage { builder in + // Build base stage + try builder + .stage(name: "base", from: alpineRef) + .run("apk add --no-cache ca-certificates") + .workdir("/app") + + // Add dependency stage + try builder + .stage(name: "deps", from: alpineRef) + .copyFromStage(.named("base"), paths: ["/etc/ssl"], to: "/etc/ssl") + .run("apk add --no-cache curl") + + // Add final stage + try builder + .stage(from: alpineRef) + .copyFromStage(.named("base"), paths: ["/app"], to: "/app") + .copyFromStage(.named("deps"), paths: ["/usr/bin/curl"], to: "/usr/local/bin/") + .entrypoint(.exec(["/app/start.sh"])) + } + + #expect(graph.stages.count == 3) + #expect(graph.stages[0].name == "base") + #expect(graph.stages[1].name == "deps") + #expect(graph.stages[2].name == nil) // Final stage + + // Verify dependencies are correct + let finalStageDeps = graph.stages[2].stageDependencies() + #expect(finalStageDeps.contains(.named("base"))) + #expect(finalStageDeps.contains(.named("deps"))) + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildIRTests/OperationTests.swift b/Tests/NativeBuilderTests/ContainerBuildIRTests/OperationTests.swift new file mode 100644 index 00000000..53621fbc --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildIRTests/OperationTests.swift @@ -0,0 +1,841 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation +import Testing + +@testable import ContainerBuildIR + +struct OperationTests { + + // MARK: - ExecOperation Tests + + @Test func execOperationBasic() throws { + let operation = ExecOperation( + command: .shell("echo 'Hello, World!'") + ) + + #expect(operation.operationKind == .exec) + #expect(operation.command.displayString == "echo 'Hello, World!'") + #expect(operation.environment.variables.isEmpty) + #expect(operation.mounts.isEmpty) + #expect(operation.workingDirectory == nil) + #expect(operation.user == nil) + #expect(operation.network == .default) + #expect(!operation.security.privileged) + } + + @Test func execOperationWithEnvironment() throws { + let environment = Environment([ + (key: "NODE_ENV", value: .literal("production")), + (key: "PORT", value: .buildArg("HTTP_PORT")), + (key: "DEBUG", value: .literal("false")), + ]) + + let operation = ExecOperation( + command: .exec(["node", "server.js"]), + environment: environment + ) + + #expect(operation.environment.variables.count == 3) + + let effectiveEnv = operation.environment.effectiveEnvironment + #expect(effectiveEnv["NODE_ENV"] == "production") + #expect(effectiveEnv["DEBUG"] == "false") + // BUILD_ARG references don't resolve without context + } + + @Test func execOperationWithMounts() throws { + let mounts = [ + Mount( + type: .cache, + target: "/var/cache/apt", + source: .local("apt-cache"), + options: MountOptions(sharing: .shared) + ), + Mount( + type: .secret, + target: "/run/secrets/github-token", + source: .secret("github-token"), + options: MountOptions(readOnly: true, mode: 0o400) + ), + Mount( + type: .tmpfs, + target: "/tmp", + source: nil, + options: MountOptions(size: 100 * 1024 * 1024) // 100MB + ), + ] + + let operation = ExecOperation( + command: .shell("apt-get update && apt-get install -y git"), + mounts: mounts + ) + + #expect(operation.mounts.count == 3) + + let cacheMount = operation.mounts[0] + #expect(cacheMount.type == .cache) + #expect(cacheMount.target == "/var/cache/apt") + #expect(cacheMount.options.sharing == .shared) + + let secretMount = operation.mounts[1] + #expect(secretMount.type == .secret) + #expect(secretMount.options.readOnly == true) + #expect(secretMount.options.mode == 0o400) + + let tmpfsMount = operation.mounts[2] + #expect(tmpfsMount.type == .tmpfs) + #expect(tmpfsMount.options.size == UInt32(100 * 1024 * 1024)) + } + + @Test func execOperationWithUser() throws { + let users = [ + User.named("appuser"), + User.uid(1000), + User.userGroup(user: "app", group: "app"), + User.uidGid(uid: 1000, gid: 1000), + ] + + for user in users { + let operation = ExecOperation( + command: .shell("whoami"), + user: user + ) + + #expect(operation.user == user) + } + } + + @Test func execOperationWithSecurity() throws { + let capabilities = SecurityCapabilities( + add: ["NET_ADMIN", "SYS_TIME"], + drop: ["ALL"] + ) + + let security = SecurityOptions( + privileged: true, + capabilities: capabilities, + seccompProfile: "custom.json", + apparmorProfile: "custom-profile", + noNewPrivileges: false + ) + + let operation = ExecOperation( + command: .shell("mount /dev/sda1 /mnt"), + security: security + ) + + #expect(operation.security.privileged == true) + #expect(operation.security.capabilities?.add.contains("NET_ADMIN") == true) + #expect(operation.security.capabilities?.drop.contains("ALL") == true) + #expect(operation.security.seccompProfile == "custom.json") + #expect(operation.security.apparmorProfile == "custom-profile") + #expect(operation.security.noNewPrivileges == false) + } + + @Test func execOperationWithNetwork() throws { + let networkModes: [NetworkMode] = [.default, .none, .host] + + for networkMode in networkModes { + let operation = ExecOperation( + command: .shell("curl https://example.com"), + network: networkMode + ) + + #expect(operation.network == networkMode) + } + } + + @Test func commandTypes() throws { + let shellCommand = Command.shell("echo 'test' && ls -la") + let execCommand = Command.exec(["ls", "-la", "/app"]) + + #expect(shellCommand.displayString == "echo 'test' && ls -la") + #expect(execCommand.displayString == "ls -la /app") + + // Test with empty exec array + let emptyExecCommand = Command.exec([]) + #expect(emptyExecCommand.displayString == "") + + // Test with single command + let singleExecCommand = Command.exec(["whoami"]) + #expect(singleExecCommand.displayString == "whoami") + } + + // MARK: - FilesystemOperation Tests + + @Test func filesystemOperationBasic() throws { + let operation = FilesystemOperation( + action: .copy, + source: .context(ContextSource(paths: ["src/"])), + destination: "/app/src/" + ) + + #expect(operation.operationKind == .filesystem) + #expect(operation.action == .copy) + #expect(operation.destination == "/app/src/") + + if case .context(let contextSource) = operation.source { + #expect(contextSource.paths == ["src/"]) + } else { + Issue.record("Expected context source") + } + } + + @Test func filesystemOperationFromStage() throws { + let operation = FilesystemOperation( + action: .copy, + source: .stage(.named("builder"), paths: ["/app/dist"]), + destination: "/usr/share/nginx/html/" + ) + + if case .stage(let stageRef, let paths) = operation.source { + if case .named(let name) = stageRef { + #expect(name == "builder") + } else { + Issue.record("Expected named stage reference") + } + #expect(paths == ["/app/dist"]) + } else { + Issue.record("Expected stage source") + } + } + + @Test func filesystemOperationWithMetadata() throws { + let expectedOwnership = Ownership(user: .numeric(id: 1000), group: .numeric(id: 1000)) + let fileMetadata = FileMetadata( + ownership: expectedOwnership, + permissions: .mode(0o644), + timestamps: Timestamps( + created: Date(), + modified: Date() + ) + ) + + let operation = FilesystemOperation( + action: .copy, + source: .context( + ContextSource( + paths: ["config.json"] + )), + destination: "/etc/app/config.json", + fileMetadata: fileMetadata + ) + + #expect(operation.fileMetadata.ownership == expectedOwnership) + + if case .mode(let mode) = operation.fileMetadata.permissions { + #expect(mode == 0o644) + } else { + Issue.record("Expected mode permissions") + } + } + + @Test func filesystemSourceTypes() throws { + let contextSource = FilesystemSource.context( + ContextSource( + paths: ["*.txt", "docs/"] + )) + + let stageSource = FilesystemSource.stage(.index(0), paths: ["/app/binary"]) + let imageSource = FilesystemSource.image( + ImageReference(parsing: "alpine:latest")!, + paths: ["/etc/ssl/certs/"] + ) + let urlSource = FilesystemSource.url(URL(string: "https://releases.example.com/v1.0.0/app.tar.gz")!) + + // Verify each source type can be created + let sources = [contextSource, stageSource, imageSource, urlSource] + for source in sources { + let operation = FilesystemOperation( + action: .copy, + source: source, + destination: "/test/" + ) + #expect(operation.source == source) + } + } + + // MARK: - MetadataOperation Tests + + @Test func metadataOperationEnvironment() throws { + let setEnvOperation = MetadataOperation( + action: .setEnv(key: "NODE_ENV", value: .literal("production")) + ) + + #expect(setEnvOperation.operationKind == .metadata) + + if case .setEnv(let key, let value) = setEnvOperation.action { + #expect(key == "NODE_ENV") + if case .literal(let literalValue) = value { + #expect(literalValue == "production") + } else { + Issue.record("Expected literal value") + } + } else { + Issue.record("Expected setEnv action") + } + } + + @Test func metadataOperationBatchEnvironment() throws { + let envVars = [ + (key: "NODE_ENV", value: EnvironmentValue.literal("production")), + (key: "PORT", value: EnvironmentValue.buildArg("HTTP_PORT")), + (key: "DEBUG", value: EnvironmentValue.literal("false")), + ] + + let operation = MetadataOperation( + action: .setEnvBatch(envVars) + ) + + if case .setEnvBatch(let vars) = operation.action { + #expect(vars.count == 3) + #expect(vars[0].key == "NODE_ENV") + #expect(vars[1].key == "PORT") + #expect(vars[2].key == "DEBUG") + } else { + Issue.record("Expected setEnvBatch action") + } + } + + @Test func metadataOperationLabels() throws { + let setLabelOperation = MetadataOperation( + action: .setLabel(key: "version", value: "1.0.0") + ) + + if case .setLabel(let key, let value) = setLabelOperation.action { + #expect(key == "version") + #expect(value == "1.0.0") + } else { + Issue.record("Expected setLabel action") + } + + let batchLabels = [ + "version": "1.0.0", + "maintainer": "team@example.com", + "description": "Sample application", + ] + + let batchLabelOperation = MetadataOperation( + action: .setLabelBatch(batchLabels) + ) + + if case .setLabelBatch(let labels) = batchLabelOperation.action { + #expect(labels.count == 3) + #expect(labels["version"] == "1.0.0") + #expect(labels["maintainer"] == "team@example.com") + } else { + Issue.record("Expected setLabelBatch action") + } + } + + @Test func metadataOperationArguments() throws { + let operation = MetadataOperation( + action: .declareArg(name: "BUILD_VERSION", defaultValue: "dev") + ) + + if case .declareArg(let name, let defaultValue) = operation.action { + #expect(name == "BUILD_VERSION") + #expect(defaultValue == "dev") + } else { + Issue.record("Expected declareArg action") + } + } + + @Test func metadataOperationExpose() throws { + let operation = MetadataOperation( + action: .expose(port: PortSpec(port: 8080, protocol: .tcp)) + ) + + if case .expose(let port) = operation.action { + #expect(port.port == 8080) + #expect(port.protocol == .tcp) + } else { + Issue.record("Expected expose action") + } + } + + @Test func metadataOperationWorkdir() throws { + let operation = MetadataOperation( + action: .setWorkdir(path: "/app") + ) + + if case .setWorkdir(let path) = operation.action { + #expect(path == "/app") + } else { + Issue.record("Expected setWorkdir action") + } + } + + @Test func metadataOperationUser() throws { + let userOperations = [ + MetadataOperation(action: .setUser(user: .named("appuser"))), + MetadataOperation(action: .setUser(user: .uid(1000))), + MetadataOperation(action: .setUser(user: .uidGid(uid: 1000, gid: 1000))), + ] + + for operation in userOperations { + if case .setUser(let user) = operation.action { + // Just verify the user is preserved + switch user { + case .named(let name): + #expect(name == "appuser") + case .uid(let uid): + #expect(uid == 1000) + case .uidGid(let uid, let gid): + #expect(uid == 1000) + #expect(gid == 1000) + default: + Issue.record("Unexpected user type") + } + } else { + Issue.record("Expected setUser action") + } + } + } + + @Test func metadataOperationCommands() throws { + let entrypointOperation = MetadataOperation( + action: .setEntrypoint(command: .exec(["/app/server"])) + ) + + let cmdOperation = MetadataOperation( + action: .setCmd(command: .shell("./start.sh")) + ) + + if case .setEntrypoint(let command) = entrypointOperation.action { + if case .exec(let args) = command { + #expect(args == ["/app/server"]) + } else { + Issue.record("Expected exec command") + } + } else { + Issue.record("Expected setEntrypoint action") + } + + if case .setCmd(let command) = cmdOperation.action { + if case .shell(let cmd) = command { + #expect(cmd == "./start.sh") + } else { + Issue.record("Expected shell command") + } + } else { + Issue.record("Expected setCmd action") + } + } + + @Test func metadataOperationHealthcheck() throws { + let healthcheck = Healthcheck( + test: .command(.exec(["curl", "-f", "http://localhost:8080/health"])), + interval: 30, + timeout: 5, + startPeriod: 10, + retries: 3 + ) + + let operation = MetadataOperation( + action: .setHealthcheck(healthcheck: healthcheck) + ) + + if case .setHealthcheck(let hc) = operation.action { + #expect(hc?.interval == 30) + #expect(hc?.timeout == 5) + #expect(hc?.startPeriod == 10) + #expect(hc?.retries == 3) + + if case .command(let cmd) = hc!.test, + case .exec(let args) = cmd + { + #expect(args == ["curl", "-f", "http://localhost:8080/health"]) + } else { + Issue.record("Expected command healthcheck test") + } + } else { + Issue.record("Expected setHealthcheck action") + } + } + + @Test func metadataOperationMiscellaneous() throws { + let stopSignalOp = MetadataOperation(action: .setStopSignal(signal: "SIGTERM")) + let volumeOp = MetadataOperation(action: .addVolume(path: "/data")) + let shellOp = MetadataOperation(action: .setShell(shell: ["/bin/bash", "-c"])) + let onBuildOp = MetadataOperation(action: .addOnBuild(instruction: "RUN npm install")) + + let operations = [stopSignalOp, volumeOp, shellOp, onBuildOp] + + for operation in operations { + #expect(operation.operationKind == .metadata) + + switch operation.action { + case .setStopSignal(let signal): + #expect(signal == "SIGTERM") + case .addVolume(let path): + #expect(path == "/data") + case .setShell(let shell): + #expect(shell == ["/bin/bash", "-c"]) + case .addOnBuild(let instruction): + #expect(instruction == "RUN npm install") + default: + Issue.record("Unexpected action type") + } + } + } + + // MARK: - ImageOperation Tests + + @Test func imageOperationFromRegistry() throws { + let imageRef = ImageReference(parsing: "alpine:3.18")! + let operation = ImageOperation( + source: .registry(imageRef), + platform: .linuxAMD64 + ) + + #expect(operation.operationKind == .image) + #expect(operation.platform == .linuxAMD64) + + if case .registry(let ref) = operation.source { + #expect(ref.stringValue == "alpine:3.18") + } else { + Issue.record("Expected registry source") + } + } + + @Test func imageOperationFromScratch() throws { + let operation = ImageOperation(source: .scratch) + + #expect(operation.platform == nil) // Default platform + + if case .scratch = operation.source { + // Expected + } else { + Issue.record("Expected scratch source") + } + } + + @Test func imageOperationFromOCILayout() throws { + let operation = ImageOperation( + source: .ociLayout(path: "/path/to/build/context", tag: "custom"), + platform: .linuxARM64 + ) + + if case .ociLayout(let path, let tag) = operation.source { + #expect(path == "/path/to/build/context") + #expect(tag == "custom") + } else { + Issue.record("Expected ociLayout source") + } + } + + // MARK: - Operation Visitor Pattern Tests + + class TestOperationVisitor: OperationVisitor { + typealias Result = String + + var visitedOperations: [String] = [] + + func visit(_ operation: ExecOperation) throws -> String { + visitedOperations.append("exec") + return "Exec: \(operation.command.displayString)" + } + + func visit(_ operation: FilesystemOperation) throws -> String { + visitedOperations.append("filesystem") + return "Filesystem: \(operation.action) to \(operation.destination)" + } + + func visit(_ operation: MetadataOperation) throws -> String { + visitedOperations.append("metadata") + return "Metadata: \(operation.action)" + } + + func visit(_ operation: ImageOperation) throws -> String { + visitedOperations.append("image") + return "Image: \(operation.source)" + } + + func visitUnknown(_ operation: any ContainerBuildIR.Operation) throws -> String { + visitedOperations.append("unknown") + return "Unknown: \(operation.operationKind)" + } + } + + @Test func operationVisitorPattern() throws { + let operations: [any ContainerBuildIR.Operation] = [ + ExecOperation(command: .shell("echo test")), + FilesystemOperation( + action: .copy, + source: .context(ContextSource(paths: ["file.txt"])), + destination: "/app/" + ), + MetadataOperation(action: .setEnv(key: "TEST", value: .literal("value"))), + ImageOperation(source: .registry(ImageReference(parsing: "alpine")!)), + ] + + let visitor = TestOperationVisitor() + var results: [String] = [] + + for operation in operations { + let result = try operation.accept(visitor) + results.append(result) + } + + #expect(visitor.visitedOperations == ["exec", "filesystem", "metadata", "image"]) + #expect(results.count == 4) + #expect(results[0].hasPrefix("Exec:")) + #expect(results[1].hasPrefix("Filesystem:")) + #expect(results[2].hasPrefix("Metadata:")) + #expect(results[3].hasPrefix("Image:")) + } + + // MARK: - Operation Serialization Tests + + @Test func operationSerialization() throws { + let operations: [any ContainerBuildIR.Operation] = [ + ExecOperation( + command: .exec(["npm", "install"]), + environment: Environment([(key: "NODE_ENV", value: .literal("production"))]), + user: .uid(1000) + ), + FilesystemOperation( + action: .copy, + source: .context(ContextSource(paths: ["src/"])), + destination: "/app/src/", + fileMetadata: FileMetadata( + ownership: Ownership(user: .numeric(id: 1000), group: .numeric(id: 1000)), + permissions: .mode(0o755) + ) + ), + MetadataOperation( + action: .setLabel(key: "version", value: "1.0.0") + ), + ImageOperation( + source: .registry(ImageReference(parsing: "node:18")!), + platform: .linuxAMD64 + ), + ] + + let encoder = JSONEncoder() + let decoder = JSONDecoder() + + for operation in operations { + // Test that each operation type can be encoded and decoded + let data: Data + + switch operation { + case let execOp as ExecOperation: + data = try encoder.encode(execOp) + let decoded = try decoder.decode(ExecOperation.self, from: data) + #expect(decoded.command.displayString == execOp.command.displayString) + #expect(decoded.user == execOp.user) + + case let fsOp as FilesystemOperation: + data = try encoder.encode(fsOp) + let decoded = try decoder.decode(FilesystemOperation.self, from: data) + #expect(decoded.action == fsOp.action) + #expect(decoded.destination == fsOp.destination) + + case let metaOp as MetadataOperation: + data = try encoder.encode(metaOp) + _ = try decoder.decode(MetadataOperation.self, from: data) + // Note: MetadataAction comparison would need custom implementation + + case let imageOp as ImageOperation: + data = try encoder.encode(imageOp) + let decoded = try decoder.decode(ImageOperation.self, from: data) + #expect(decoded.platform == imageOp.platform) + + default: + Issue.record("Unexpected operation type") + } + } + } + + // MARK: - Operation Metadata Tests + + @Test func operationMetadata() throws { + let sourceLocation = SourceLocation( + file: "Dockerfile", + line: 15, + column: 5 + ) + + let metadata = OperationMetadata( + comment: "Install application dependencies", + sourceLocation: sourceLocation + ) + + let operation = ExecOperation( + command: .shell("npm install"), + metadata: metadata + ) + + #expect(operation.metadata.comment == "Install application dependencies") + #expect(operation.metadata.sourceLocation?.file == "Dockerfile") + #expect(operation.metadata.sourceLocation?.line == 15) + } + + // MARK: - Complex Operation Tests + + @Test func complexExecOperation() throws { + let complexOperation = ExecOperation( + command: .shell("cd /app && npm ci --only=production && npm run build"), + environment: Environment([ + (key: "NODE_ENV", value: .literal("production")), + (key: "BUILD_TARGET", value: .buildArg("TARGET")), + (key: "CACHE_DIR", value: .literal("/tmp/cache")), + ]), + mounts: [ + Mount( + type: .cache, + target: "/tmp/cache", + source: .local("build-cache"), + options: MountOptions(sharing: .shared) + ), + Mount( + type: .secret, + target: "/run/secrets/npmrc", + source: .secret("npmrc"), + options: MountOptions(readOnly: true, mode: 0o600) + ), + ], + workingDirectory: "/app", + user: .uidGid(uid: 1000, gid: 1000), + network: .default, + security: SecurityOptions( + privileged: false, + noNewPrivileges: true + ), + metadata: OperationMetadata( + comment: "Build application with caching and secrets" + ) + ) + + #expect(complexOperation.command.displayString.contains("npm ci")) + #expect(complexOperation.environment.variables.count == 3) + #expect(complexOperation.mounts.count == 2) + #expect(complexOperation.workingDirectory == "/app") + #expect(complexOperation.user != nil) + #expect(complexOperation.security.noNewPrivileges == true) + #expect(complexOperation.metadata.comment?.contains("Build application") == true) + } + + @Test func complexFilesystemOperation() throws { + let complexOperation = FilesystemOperation( + action: .copy, + source: .stage(.named("builder"), paths: ["/app/dist/**/*", "/app/package.json"]), + destination: "/usr/share/nginx/html/", + fileMetadata: FileMetadata( + ownership: Ownership(user: .named(id: "nginx"), group: .named(id: "nginx")), + permissions: .mode(0o644), + timestamps: Timestamps( + created: Date(), + modified: Date() + ) + ), + metadata: OperationMetadata( + comment: "Copy built assets from builder stage", + sourceLocation: SourceLocation(file: "Dockerfile", line: 25, column: 1) + ) + ) + + if case .stage(let stageRef, let paths) = complexOperation.source { + if case .named(let name) = stageRef { + #expect(name == "builder") + } + #expect(paths.count == 2) + #expect(paths.contains("/app/dist/**/*")) + } else { + Issue.record("Expected stage source") + } + + #expect(complexOperation.destination == "/usr/share/nginx/html/") + // Check fileMetadata exists and has expected values + #expect(complexOperation.metadata.comment?.contains("built assets") == true) + } + + // MARK: - Edge Cases and Error Conditions + + @Test func emptyOperations() throws { + // Test operations with minimal/empty configurations + let minimalExec = ExecOperation(command: .exec([])) + #expect(minimalExec.command.displayString == "") + #expect(minimalExec.mounts.isEmpty) + + let minimalFilesystem = FilesystemOperation( + action: .copy, + source: .context(ContextSource(paths: [])), + destination: "" + ) + #expect(minimalFilesystem.destination == "") + + if case .context(let ctx) = minimalFilesystem.source { + #expect(ctx.paths.isEmpty) + } + } + + @Test func operationEquality() throws { + let exec1 = ExecOperation(command: .shell("echo test")) + let exec2 = ExecOperation(command: .shell("echo test")) + let exec3 = ExecOperation(command: .shell("echo different")) + + #expect(exec1 == exec2) + #expect(exec1 != exec3) + + let fs1 = FilesystemOperation( + action: .copy, + source: .context(ContextSource(paths: ["file.txt"])), + destination: "/app/" + ) + let fs2 = FilesystemOperation( + action: .copy, + source: .context(ContextSource(paths: ["file.txt"])), + destination: "/app/" + ) + + #expect(fs1 == fs2) + } + + @Test func operationHashing() throws { + let exec1 = ExecOperation(command: .shell("echo test")) + let exec2 = ExecOperation(command: .shell("echo different")) + let fs1 = FilesystemOperation( + action: .copy, + source: .context(ContextSource(paths: ["file.txt"])), + destination: "/app/" + ) + + let operations: Set = Set([ + AnyHashable(exec1), + AnyHashable(exec2), + AnyHashable(fs1), + ]) + + #expect(operations.count == 3) + + // Test that identical operations hash to the same value + let exec1_copy = ExecOperation(command: .shell("echo test")) + let exec2_copy = ExecOperation(command: .shell("echo test")) + + var hasher1 = Hasher() + var hasher2 = Hasher() + + exec1_copy.hash(into: &hasher1) + exec2_copy.hash(into: &hasher2) + + #expect(hasher1.finalize() == hasher2.finalize()) + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildIRTests/SerializationTests.swift b/Tests/NativeBuilderTests/ContainerBuildIRTests/SerializationTests.swift new file mode 100644 index 00000000..1ffca99f --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildIRTests/SerializationTests.swift @@ -0,0 +1,620 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation +import Testing + +@testable import ContainerBuildIR + +struct SerializationTests { + + // MARK: - JSONIRCoder Tests + + @Test func jsonCoderBasicRoundTrip() throws { + guard let imageRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + let originalGraph = try GraphBuilder.singleStage(from: imageRef) { builder in + try builder + .run("apk add --no-cache curl") + .workdir("/app") + .copyFromContext(paths: ["src/"], to: "/app/") + .env("NODE_ENV", "production") + .expose(3000) + .cmd(.exec(["node", "index.js"])) + } + + let coder = JSONIRCoder() + let encodedData = try coder.encode(originalGraph) + let decodedGraph = try coder.decode(encodedData) + + // Verify structure preservation + #expect(decodedGraph.stages.count == originalGraph.stages.count) + #expect(decodedGraph.buildArgs == originalGraph.buildArgs) + #expect(decodedGraph.targetPlatforms == originalGraph.targetPlatforms) + + // Verify stage content + let originalStage = originalGraph.stages[0] + let decodedStage = decodedGraph.stages[0] + #expect(decodedStage.nodes.count == originalStage.nodes.count) + #expect(decodedStage.name == originalStage.name) + } + + @Test func jsonCoderPrettyPrint() throws { + guard let imageRef = ImageReference(parsing: "ubuntu:22.04") else { + Issue.record("Failed to parse image reference") + return + } + + let graph = try GraphBuilder.singleStage(from: imageRef) { builder in + try builder + .run("apt-get update") + .run("apt-get install -y python3") + } + + let prettyPrintCoder = JSONIRCoder(prettyPrint: true) + let compactCoder = JSONIRCoder(prettyPrint: false) + + let prettyData = try prettyPrintCoder.encode(graph) + let compactData = try compactCoder.encode(graph) + + // Pretty printed should be larger (contains whitespace) + #expect(prettyData.count > compactData.count) + + // Both should decode to the same graph + let prettyGraph = try prettyPrintCoder.decode(prettyData) + let compactGraph = try compactCoder.decode(compactData) + + #expect(prettyGraph.stages.count == compactGraph.stages.count) + #expect(prettyGraph.buildArgs == compactGraph.buildArgs) + + // Verify JSON is actually pretty printed + let prettyJson = String(data: prettyData, encoding: .utf8)! + #expect(prettyJson.contains("\n"), "Pretty printed JSON should contain newlines") + #expect(prettyJson.contains(" "), "Pretty printed JSON should contain indentation") + } + + @Test func jsonCoderMultiStageGraph() throws { + guard let nodeRef = ImageReference(parsing: "node:18"), + let nginxRef = ImageReference(parsing: "nginx:alpine") + else { + Issue.record("Failed to parse image references") + return + } + + let originalGraph = try GraphBuilder.multiStage { builder in + try builder + .stage(name: "build", from: nodeRef) + .workdir("/app") + .copyFromContext(paths: ["package.json"], to: "./") + .run("npm install") + .copyFromContext(paths: ["src/"], to: "./src/") + .run("npm run build") + + try builder + .stage(from: nginxRef) + .copyFromStage(.named("build"), paths: ["/app/dist"], to: "/usr/share/nginx/html") + .expose(80) + } + + let coder = JSONIRCoder() + let encodedData = try coder.encode(originalGraph) + let decodedGraph = try coder.decode(encodedData) + + #expect(decodedGraph.stages.count == 2) + #expect(decodedGraph.stages[0].name == "build") + #expect(decodedGraph.stages[1].name == nil) + + // Verify stage dependencies are preserved + let runtimeStage = decodedGraph.stages[1] + let stageDeps = runtimeStage.stageDependencies() + #expect(stageDeps.contains(.named("build"))) + } + + @Test func jsonCoderWithBuildArgs() throws { + guard let nodeRef = ImageReference(parsing: "node:18") else { + Issue.record("Failed to parse image reference") + return + } + + let originalGraph = try GraphBuilder.singleStage(from: nodeRef) { builder in + try builder + .arg("NODE_ENV", defaultValue: "development") + .arg("BUILD_VERSION") + .env("NODE_ENV", "production") + .env("BUILD_VERSION", "1.2.3") + .run("npm install") + } + + let coder = JSONIRCoder() + let encodedData = try coder.encode(originalGraph) + let decodedGraph = try coder.decode(encodedData) + + // Verify build args preservation + #expect(decodedGraph.buildArgs.count >= 0) + + // Verify target platforms preservation + #expect(decodedGraph.targetPlatforms.count >= 0) + } + + // MARK: - BinaryIRCoder Tests + + @Test func binaryCoderBasicRoundTrip() throws { + guard let imageRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + let originalGraph = try GraphBuilder.singleStage(from: imageRef) { builder in + try builder + .run("apk add --no-cache git") + .workdir("/app") + .copyFromContext(paths: ["main.go"], to: "/app/") + .run("go build -o app main.go") + .entrypoint(.exec(["/app/app"])) + } + + let coder = BinaryIRCoder() + let encodedData = try coder.encode(originalGraph) + let decodedGraph = try coder.decode(encodedData) + + // Verify structure preservation + #expect(decodedGraph.stages.count == originalGraph.stages.count) + #expect(decodedGraph.buildArgs == originalGraph.buildArgs) + #expect(decodedGraph.targetPlatforms == originalGraph.targetPlatforms) + + // Verify stage content + let originalStage = originalGraph.stages[0] + let decodedStage = decodedGraph.stages[0] + #expect(decodedStage.nodes.count == originalStage.nodes.count) + } + + @Test func binaryCoderCompression() throws { + guard let ubuntuRef = ImageReference(parsing: "ubuntu:22.04") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a larger graph to test compression effectiveness + let originalGraph = try GraphBuilder.multiStage { builder in + for i in 0..<3 { + try builder + .stage(name: "stage\(i)", from: ubuntuRef) + .run("apt-get update") + .run("apt-get install -y curl wget git vim") + .workdir("/app") + .copyFromContext(paths: ["file\(i).txt"], to: "/app/") + .env("STAGE_NUMBER", "\(i)") + .label("stage.number", "\(i)") + .label("stage.description", "This is stage number \(i) with detailed information") + } + } + + let jsonCoder = JSONIRCoder(prettyPrint: false) + let binaryCoder = BinaryIRCoder() + + let jsonData = try jsonCoder.encode(originalGraph) + let binaryData = try binaryCoder.encode(originalGraph) + + // Binary format should typically be smaller than JSON + #expect( + binaryData.count <= jsonData.count, + "Binary format should be more compact (JSON: \(jsonData.count), Binary: \(binaryData.count))") + + // Verify both decode correctly + let jsonGraph = try jsonCoder.decode(jsonData) + let binaryGraph = try binaryCoder.decode(binaryData) + + #expect(jsonGraph.stages.count == binaryGraph.stages.count) + #expect(jsonGraph.buildArgs == binaryGraph.buildArgs) + } + + @Test func binaryCoderComplexOperations() throws { + guard let alpineRef = ImageReference(parsing: "alpine:latest") else { + Issue.record("Failed to parse image reference") + return + } + + let originalGraph = try GraphBuilder.singleStage(from: alpineRef) { builder in + try builder + .run( + "apk add --no-cache build-tools", + mounts: [ + Mount(type: .cache, target: "/var/cache/apk", source: .local("apk-cache")), + Mount(type: .secret, target: "/run/secrets/token", source: .secret("github-token")), + ] + ) + .copyFromContext( + paths: ["src/**/*.c", "include/**/*.h"], + to: "/app/" + ) + .healthcheck( + test: .command(.exec(["./healthcheck.sh"])), + interval: 30, + timeout: 5, + retries: 3 + ) + .user(.uidGid(uid: 1000, gid: 1000)) + } + + let coder = BinaryIRCoder() + let encodedData = try coder.encode(originalGraph) + let decodedGraph = try coder.decode(encodedData) + + // Verify complex operations are preserved + let stage = decodedGraph.stages[0] + + // Check for RUN operation with mounts + let runOps = stage.nodes.compactMap { $0.operation as? ExecOperation } + let runWithMounts = runOps.first { !$0.mounts.isEmpty } + #expect(runWithMounts != nil, "RUN operation with mounts should be preserved") + #expect(runWithMounts!.mounts.count == 2, "Both mounts should be preserved") + + // Check for COPY operation with metadata + let copyOps = stage.nodes.compactMap { $0.operation as? FilesystemOperation } + let copyWithMetadata = copyOps.first { $0.fileMetadata != nil } + #expect(copyWithMetadata != nil, "COPY operation with metadata should be preserved") + + // Check for metadata operations + let metaOps = stage.nodes.compactMap { $0.operation as? MetadataOperation } + let hasHealthcheck = metaOps.contains { + if case .setHealthcheck = $0.action { return true } + return false + } + #expect(hasHealthcheck, "Healthcheck metadata should be preserved") + } + + // MARK: - Format Comparison Tests + + @Test func formatSizeComparison() throws { + guard let golangRef = ImageReference(parsing: "golang:1.21"), + let alpineRef = ImageReference(parsing: "alpine:latest") + else { + Issue.record("Failed to parse image references") + return + } + + // Create a realistic build graph + let graph = try GraphBuilder.multiStage { builder in + try builder + .stage(name: "builder", from: golangRef) + .workdir("/src") + .copyFromContext(paths: ["go.mod", "go.sum"], to: "./") + .run("go mod download") + .copyFromContext(paths: ["cmd/", "internal/", "pkg/"], to: "./") + .run("CGO_ENABLED=0 GOOS=linux go build -ldflags='-w -s' -o /app cmd/main.go") + + try builder + .stage(from: alpineRef) + .run("apk add --no-cache ca-certificates") + .copyFromStage(.named("builder"), paths: ["/app"], to: "/usr/local/bin/") + .user(.uid(65534)) + .expose(8080) + .entrypoint(.exec(["/usr/local/bin/app"])) + } + + let jsonCompactCoder = JSONIRCoder(prettyPrint: false) + let jsonPrettyCoder = JSONIRCoder(prettyPrint: true) + let binaryCoder = BinaryIRCoder() + + let jsonCompactData = try jsonCompactCoder.encode(graph) + let jsonPrettyData = try jsonPrettyCoder.encode(graph) + let binaryData = try binaryCoder.encode(graph) + + print("Format size comparison:") + print("- JSON (compact): \(jsonCompactData.count) bytes") + print("- JSON (pretty): \(jsonPrettyData.count) bytes") + print("- Binary: \(binaryData.count) bytes") + + // Expected size ordering + #expect(binaryData.count <= jsonCompactData.count) + #expect(jsonCompactData.count < jsonPrettyData.count) + + // All should decode to equivalent graphs + let graphs = [ + try jsonCompactCoder.decode(jsonCompactData), + try jsonPrettyCoder.decode(jsonPrettyData), + try binaryCoder.decode(binaryData), + ] + + for graph in graphs { + #expect(graph.stages.count == 2) + #expect(graph.stages[0].name == "builder") + } + } + + // MARK: - Error Handling Tests + + @Test func corruptedDataHandling() throws { + let coder = JSONIRCoder() + + // Test completely invalid data + let invalidData = "not valid json".data(using: .utf8)! + #expect(throws: Error.self) { + try coder.decode(invalidData) + } + + // Test valid JSON but invalid structure + let invalidStructure = """ + { + "version": "1.0", + "graph": { + "stages": "this should be an array" + } + } + """.data(using: .utf8)! + + #expect(throws: Error.self) { + try coder.decode(invalidStructure) + } + } + + @Test func binaryCorruptedDataHandling() throws { + let coder = BinaryIRCoder() + + // Test invalid binary data + let invalidData = Data(repeating: 0xFF, count: 100) + #expect(throws: Error.self) { + try coder.decode(invalidData) + } + + // Test truncated data + let validGraph: BuildGraph + do { + validGraph = try GraphBuilder.singleStage( + from: ImageReference(parsing: "alpine")! + ) { builder in + try builder.run("echo test") + } + } catch { + Issue.record("Failed to create test graph") + return + } + + let validData = try coder.encode(validGraph) + let truncatedData = validData.prefix(validData.count / 2) + + #expect(throws: Error.self) { + try coder.decode(Data(truncatedData)) + } + } + + @Test func versionHandling() throws { + // Test that we can detect version information in JSON format + let graph: BuildGraph + do { + graph = try GraphBuilder.singleStage( + from: ImageReference(parsing: "alpine")! + ) { builder in + try builder.run("echo version test") + } + } catch { + Issue.record("Failed to create test graph") + return + } + + let coder = JSONIRCoder(prettyPrint: true) + let data = try coder.encode(graph) + let jsonString = String(data: data, encoding: .utf8)! + + #expect(jsonString.contains("\"version\""), "JSON should contain version information") + #expect(jsonString.contains("\"1.0\""), "Should use version 1.0") + + // Should decode correctly + let decodedGraph = try coder.decode(data) + #expect(decodedGraph.stages.count == graph.stages.count) + } + + // MARK: - File I/O Tests + + @Test func saveAndLoadGraph() throws { + guard let imageRef = ImageReference(parsing: "python:3.11") else { + Issue.record("Failed to parse image reference") + return + } + + let originalGraph = try GraphBuilder.singleStage(from: imageRef) { builder in + try builder + .workdir("/app") + .copyFromContext(paths: ["requirements.txt"], to: "./") + .run("pip install -r requirements.txt") + .copyFromContext(paths: ["src/"], to: "./src/") + .cmd(.exec(["python", "src/main.py"])) + } + + let tempDir = FileManager.default.temporaryDirectory + let jsonURL = tempDir.appendingPathComponent("test-graph.json") + let binaryURL = tempDir.appendingPathComponent("test-graph.bin") + + // Save using different formats + try originalGraph.save(to: jsonURL, using: JSONIRCoder(prettyPrint: true)) + try originalGraph.save(to: binaryURL, using: BinaryIRCoder()) + + // Verify files exist and have content + #expect(FileManager.default.fileExists(atPath: jsonURL.path)) + #expect(FileManager.default.fileExists(atPath: binaryURL.path)) + + let jsonFileSize = try FileManager.default.attributesOfItem(atPath: jsonURL.path)[.size] as! Int + let binaryFileSize = try FileManager.default.attributesOfItem(atPath: binaryURL.path)[.size] as! Int + + #expect(jsonFileSize > 0) + #expect(binaryFileSize > 0) + + // Load and verify + let jsonGraph = try BuildGraph.load(from: jsonURL, using: JSONIRCoder()) + let binaryGraph = try BuildGraph.load(from: binaryURL, using: BinaryIRCoder()) + + #expect(jsonGraph.stages.count == originalGraph.stages.count) + #expect(binaryGraph.stages.count == originalGraph.stages.count) + + // Cleanup + try? FileManager.default.removeItem(at: jsonURL) + try? FileManager.default.removeItem(at: binaryURL) + } + + // MARK: - Performance Tests + + @Test func serializationPerformance() throws { + // Create a large graph for performance testing + guard let baseRef = ImageReference(parsing: "ubuntu:22.04") else { + Issue.record("Failed to parse image reference") + return + } + + var stages: [BuildStage] = [] + + // Create 5 stages with 10 operations each + for stageIndex in 0..<5 { + var nodes: [BuildNode] = [] + + for opIndex in 0..<10 { + let operation: any ContainerBuildIR.Operation + switch opIndex % 4 { + case 0: + operation = ExecOperation(command: .shell("echo 'stage \(stageIndex) op \(opIndex)'")) + case 1: + operation = FilesystemOperation( + action: .copy, + source: .context(ContextSource(paths: ["file\(opIndex).txt"])), + destination: "/app/file\(opIndex).txt" + ) + case 2: + operation = MetadataOperation(action: .setEnv(key: "VAR\(opIndex)", value: .literal("value\(opIndex)"))) + default: + operation = MetadataOperation(action: .setLabel(key: "label\(opIndex)", value: "value\(opIndex)")) + } + + nodes.append(BuildNode(operation: operation, dependencies: [])) + } + + stages.append( + BuildStage( + name: "stage\(stageIndex)", + base: ImageOperation(source: .registry(baseRef)), + nodes: nodes + )) + } + + let largeGraph = try BuildGraph(stages: stages) + + let jsonCoder = JSONIRCoder() + let binaryCoder = BinaryIRCoder() + + // Measure JSON encoding time + let jsonStartTime = Date() + let jsonData = try jsonCoder.encode(largeGraph) + let jsonEncodeTime = Date().timeIntervalSince(jsonStartTime) + + // Measure binary encoding time + let binaryStartTime = Date() + let binaryData = try binaryCoder.encode(largeGraph) + let binaryEncodeTime = Date().timeIntervalSince(binaryStartTime) + + print("Encoding performance for large graph:") + print("- JSON: \(String(format: "%.3f", jsonEncodeTime))s (\(jsonData.count) bytes)") + print("- Binary: \(String(format: "%.3f", binaryEncodeTime))s (\(binaryData.count) bytes)") + + // Both should complete in reasonable time (under 1 second for this size) + #expect(jsonEncodeTime < 1.0, "JSON encoding should be fast") + #expect(binaryEncodeTime < 1.0, "Binary encoding should be fast") + + // Measure decoding performance + let jsonDecodeStart = Date() + let _ = try jsonCoder.decode(jsonData) + let jsonDecodeTime = Date().timeIntervalSince(jsonDecodeStart) + + let binaryDecodeStart = Date() + let _ = try binaryCoder.decode(binaryData) + let binaryDecodeTime = Date().timeIntervalSince(binaryDecodeStart) + + print("Decoding performance for large graph:") + print("- JSON: \(String(format: "%.3f", jsonDecodeTime))s") + print("- Binary: \(String(format: "%.3f", binaryDecodeTime))s") + + #expect(jsonDecodeTime < 1.0, "JSON decoding should be fast") + #expect(binaryDecodeTime < 1.0, "Binary decoding should be fast") + } + + // MARK: - Edge Cases + + @Test func emptyGraphSerialization() throws { + // Test edge case of empty graph + let emptyGraph = try BuildGraph(stages: []) + + let jsonCoder = JSONIRCoder() + let binaryCoder = BinaryIRCoder() + + let jsonData = try jsonCoder.encode(emptyGraph) + let binaryData = try binaryCoder.encode(emptyGraph) + + let jsonDecoded = try jsonCoder.decode(jsonData) + let binaryDecoded = try binaryCoder.decode(binaryData) + + #expect(jsonDecoded.stages.isEmpty) + #expect(binaryDecoded.stages.isEmpty) + #expect(jsonDecoded.buildArgs.isEmpty) + #expect(binaryDecoded.buildArgs.isEmpty) + } + + @Test func graphWithSpecialCharacters() throws { + guard let baseRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let graph = try GraphBuilder.singleStage(from: baseRef) { builder in + try builder + .run("echo 'Special chars: 中文 émojis 🚀 newlines\n and quotes \"test\"'") + .env("UNICODE", "Contains unicode: ñáéíóú 中文字符 🌍") + .label("description", "Multi-line\ndescription with\ttabs and spaces") + .workdir("/app with spaces/and-symbols!@#$%") + } + + let jsonCoder = JSONIRCoder(prettyPrint: true) + let binaryCoder = BinaryIRCoder() + + let jsonData = try jsonCoder.encode(graph) + let binaryData = try binaryCoder.encode(graph) + + let jsonDecoded = try jsonCoder.decode(jsonData) + let binaryDecoded = try binaryCoder.decode(binaryData) + + #expect(jsonDecoded.stages.count == 1) + #expect(binaryDecoded.stages.count == 1) + + // Check that the operations contain the expected special characters + let stage = jsonDecoded.stages[0] + let execOp = stage.nodes.compactMap { $0.operation as? ExecOperation }.first! + let envOp = stage.nodes.compactMap { $0.operation as? MetadataOperation }.first { op in + if case .setEnv(let key, _) = op.action, key == "UNICODE" { return true } + return false + }! + + // Verify special characters are preserved + if case .shell(let command) = execOp.command { + #expect(command.contains("中文"), "Unicode characters should be preserved in command") + #expect(command.contains("🚀"), "Emoji should be preserved in command") + #expect(command.contains("\n"), "Newlines should be preserved in command") + } + + if case .setEnv(_, let value) = envOp.action, case .literal(let envValue) = value { + #expect(envValue.contains("中文字符"), "Unicode characters should be preserved in env") + #expect(envValue.contains("🌍"), "Emoji should be preserved in env") + } + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildIRTests/ValidationTests.swift b/Tests/NativeBuilderTests/ContainerBuildIRTests/ValidationTests.swift new file mode 100644 index 00000000..a96211f5 --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildIRTests/ValidationTests.swift @@ -0,0 +1,928 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation +import Testing + +@testable import ContainerBuildIR + +struct ValidationTests { + + // MARK: - StructuralValidator Tests + + @Test func duplicateNodeIDs() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let duplicateId = UUID() + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + id: duplicateId, + operation: ExecOperation(command: .shell("echo 'first'")), + dependencies: [] + ), + BuildNode( + id: duplicateId, // Duplicate ID + operation: ExecOperation(command: .shell("echo 'second'")), + dependencies: [] + ), + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let validator = StructuralValidator() + let result = validator.validate(graph) + + #expect(!result.isValid) + + let hasDuplicateError = result.errors.contains { error in + if case .duplicateNodeID(let id, _) = error { + return id == duplicateId + } + return false + } + #expect(hasDuplicateError, "Should detect duplicate node ID") + } + + @Test func missingDependency() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let missingId = UUID() + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: ExecOperation(command: .shell("echo 'dependent'")), + dependencies: Set([missingId]) // References non-existent node + ) + ] + ) + + // BuildGraph constructor should throw for invalid dependencies + #expect(throws: BuildGraphError.self) { + try BuildGraph(stages: [stage]) + } + } + + @Test func validStructuralGraph() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let node1 = BuildNode( + operation: ExecOperation(command: .shell("echo 'first'")), + dependencies: [] + ) + let node2 = BuildNode( + operation: ExecOperation(command: .shell("echo 'second'")), + dependencies: Set([node1.id]) + ) + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [node1, node2] + ) + + let graph = try BuildGraph(stages: [stage]) + let validator = StructuralValidator() + let result = validator.validate(graph) + + #expect(result.isValid) + #expect(result.errors.isEmpty) + } + + // MARK: - ReferenceValidator Tests + + @Test func undefinedStageReference() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .stage(.named("nonexistent"), paths: ["/file"]), + destination: "/test/file" + ), + dependencies: [] + ) + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let validator = ReferenceValidator() + let result = validator.validate(graph) + + #expect(!result.isValid) + + let hasUndefinedStageError = result.errors.contains { error in + if case .undefinedStageReference(let name, _) = error { + return name == "nonexistent" + } + return false + } + #expect(hasUndefinedStageError, "Should detect undefined stage reference") + } + + @Test func stageIndexOutOfBounds() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .stage(.index(99), paths: ["/file"]), // Out of bounds + destination: "/test/file" + ), + dependencies: [] + ) + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let validator = ReferenceValidator() + let result = validator.validate(graph) + + #expect(!result.isValid) + + let hasOutOfBoundsError = result.errors.contains { error in + if case .stageIndexOutOfBounds(let index, _) = error { + return index == 99 + } + return false + } + #expect(hasOutOfBoundsError, "Should detect stage index out of bounds") + } + + @Test func invalidPreviousReference() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + // First stage cannot reference "previous" + let stage = BuildStage( + name: "first", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .stage(.previous, paths: ["/file"]), + destination: "/test/file" + ), + dependencies: [] + ) + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let validator = ReferenceValidator() + let result = validator.validate(graph) + + #expect(!result.isValid) + + let hasInvalidPrevError = result.errors.contains { error in + if case .invalidPreviousReference = error { + return true + } + return false + } + #expect(hasInvalidPrevError, "Should detect invalid previous reference in first stage") + } + + @Test func forwardStageReferenceWarnings() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let stage1 = BuildStage( + name: "first", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .stage(.named("second"), paths: ["/file"]), // Forward reference + destination: "/test/file" + ), + dependencies: [] + ) + ] + ) + + let stage2 = BuildStage( + name: "second", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: ExecOperation(command: .shell("echo 'second stage'")), + dependencies: [] + ) + ] + ) + + let graph = try BuildGraph(stages: [stage1, stage2]) + let validator = ReferenceValidator() + let result = validator.validate(graph) + + // Should be structurally valid but have warnings + #expect(result.isValid) + #expect(!result.warnings.isEmpty) + + let hasForwardRefWarning = result.warnings.contains { warning in + if case .forwardStageReferenceByName(let name, _) = warning { + return name == "second" + } + return false + } + #expect(hasForwardRefWarning, "Should warn about forward stage reference") + } + + // MARK: - PathValidator Tests + + @Test func emptyDestinationPath() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .context(ContextSource(paths: ["file.txt"])), + destination: "" // Empty destination + ), + dependencies: [] + ) + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let validator = PathValidator() + let result = validator.validate(graph) + + #expect(!result.isValid) + + let hasEmptyDestError = result.errors.contains { error in + if case .emptyDestinationPath = error { + return true + } + return false + } + #expect(hasEmptyDestError, "Should detect empty destination path") + } + + @Test func absoluteContextPath() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .context(ContextSource(paths: ["/absolute/path"])), // Absolute path + destination: "/app/" + ), + dependencies: [] + ) + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let validator = PathValidator() + let result = validator.validate(graph) + + #expect(!result.isValid) + + let hasAbsolutePathError = result.errors.contains { error in + if case .absoluteContextPath(let path, _) = error { + return path == "/absolute/path" + } + return false + } + #expect(hasAbsolutePathError, "Should detect absolute context path") + } + + @Test func pathWithDotDotWarning() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: FilesystemOperation( + action: .copy, + source: .context(ContextSource(paths: ["../outside/file.txt"])), + destination: "/app/" + ), + dependencies: [] + ) + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let validator = PathValidator() + let result = validator.validate(graph) + + #expect(result.isValid) // Should be valid but have warnings + #expect(!result.warnings.isEmpty) + + let hasDotDotWarning = result.warnings.contains { warning in + if case .pathContainsDotDot(let path, _) = warning { + return path == "../outside/file.txt" + } + return false + } + #expect(hasDotDotWarning, "Should warn about path containing '..'") + } + + @Test func emptyMountTarget() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: ExecOperation( + command: .shell("echo 'test'"), + mounts: [ + Mount( + type: .cache, + target: nil, // Empty target + source: .local("cache-vol"), + options: MountOptions() + ) + ] + ), + dependencies: [] + ) + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let validator = PathValidator() + let result = validator.validate(graph) + + #expect(!result.isValid) + + let hasEmptyMountError = result.errors.contains { error in + if case .emptyMountTarget = error { + return true + } + return false + } + #expect(hasEmptyMountError, "Should detect empty mount target") + } + + // MARK: - SecurityValidator Tests + + @Test func privilegedExecutionWarning() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: ExecOperation( + command: .shell("mount /dev/sda1 /mnt"), + security: SecurityOptions(privileged: true) + ), + dependencies: [] + ) + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let validator = SecurityValidator() + let result = validator.validate(graph) + + #expect(result.isValid) // Should be valid but have warnings + #expect(!result.warnings.isEmpty) + + let hasPrivilegedWarning = result.warnings.contains { warning in + if case .privilegedExecution = warning { + return true + } + return false + } + #expect(hasPrivilegedWarning, "Should warn about privileged execution") + } + + @Test func runningAsRootWarning() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: ExecOperation( + command: .shell("apt-get update"), + user: nil // Running as root + ), + dependencies: [] + ) + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let validator = SecurityValidator() + let result = validator.validate(graph) + + #expect(result.isValid) // Should be valid but have warnings + #expect(!result.warnings.isEmpty) + + let hasRootWarning = result.warnings.contains { warning in + if case .runningAsRoot = warning { + return true + } + return false + } + #expect(hasRootWarning, "Should warn about running as root") + } + + @Test func readWriteSecretMountWarning() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: ExecOperation( + command: .shell("cat /run/secrets/token"), + mounts: [ + Mount( + type: .secret, + target: "/run/secrets/token", + source: .secret("api-token"), + options: MountOptions(readOnly: false) // Read-write secret + ) + ] + ), + dependencies: [] + ) + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let validator = SecurityValidator() + let result = validator.validate(graph) + + #expect(result.isValid) // Should be valid but have warnings + #expect(!result.warnings.isEmpty) + + let hasSecretWarning = result.warnings.contains { warning in + if case .readWriteSecretMount = warning { + return true + } + return false + } + #expect(hasSecretWarning, "Should warn about read-write secret mount") + } + + // MARK: - BestPracticesValidator Tests + + @Test func aptUpdateWithoutInstallWarning() throws { + guard let ubuntuRef = ImageReference(parsing: "ubuntu") else { + Issue.record("Failed to parse image reference") + return + } + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .registry(ubuntuRef)), + nodes: [ + BuildNode( + operation: ExecOperation(command: .shell("apt-get update")), + dependencies: [] + ) + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let validator = BestPracticesValidator() + let result = validator.validate(graph) + + #expect(result.isValid) // Should be valid but have warnings + #expect(!result.warnings.isEmpty) + + let hasAptUpdateWarning = result.warnings.contains { warning in + if case .aptGetUpdateWithoutInstall = warning { + return true + } + return false + } + #expect(hasAptUpdateWarning, "Should warn about apt-get update without install") + } + + @Test func missingHealthcheckWarning() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + // Build without healthcheck in target stage + let stage = BuildStage( + name: "app", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: ExecOperation(command: .shell("apk add --no-cache curl")), + dependencies: [] + ), + BuildNode( + operation: MetadataOperation(action: .setEntrypoint(command: .exec(["./app"]))), + dependencies: [] + ), + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let validator = BestPracticesValidator() + let result = validator.validate(graph) + + #expect(result.isValid) // Should be valid but have warnings + #expect(!result.warnings.isEmpty) + + let hasMissingHealthcheckWarning = result.warnings.contains { warning in + if case .missingHealthcheck = warning { + return true + } + return false + } + #expect(hasMissingHealthcheckWarning, "Should warn about missing healthcheck") + } + + @Test func validBestPractices() throws { + guard let ubuntuRef = ImageReference(parsing: "ubuntu") else { + Issue.record("Failed to parse image reference") + return + } + + let stage = BuildStage( + name: "app", + base: ImageOperation(source: .registry(ubuntuRef)), + nodes: [ + BuildNode( + operation: ExecOperation(command: .shell("apt-get update && apt-get install -y curl")), + dependencies: [] + ), + BuildNode( + operation: MetadataOperation( + action: .setHealthcheck( + healthcheck: Healthcheck( + test: .command(.exec(["curl", "-f", "http://localhost:8080/health"])), + interval: 30, + timeout: 5, + startPeriod: nil, + retries: 3 + ) + ) + ), + dependencies: [] + ), + BuildNode( + operation: MetadataOperation(action: .setEntrypoint(command: .exec(["./app"]))), + dependencies: [] + ), + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let validator = BestPracticesValidator() + let result = validator.validate(graph) + + #expect(result.isValid) + #expect(result.warnings.isEmpty, "Should not have warnings when following best practices") + } + + // MARK: - CompositeValidator Tests + + @Test func compositeValidatorCombinesResults() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a graph with multiple types of issues + let duplicateId = UUID() + let stage = BuildStage( + name: "problematic", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + id: duplicateId, + operation: ExecOperation(command: .shell("echo 'first'")), + dependencies: [] + ), + BuildNode( + id: duplicateId, // Duplicate ID (structural error) + operation: FilesystemOperation( + action: .copy, + source: .stage(.named("missing"), paths: ["/file"]), // Missing stage (reference error) + destination: "" // Empty destination (path error) + ), + dependencies: [] + ), + BuildNode( + operation: ExecOperation( + command: .shell("apt-get update"), // Update without install (best practices warning) + security: SecurityOptions(privileged: true) // Privileged (security warning) + ), + dependencies: [] + ), + ] + ) + + let graph = try BuildGraph(stages: [stage]) + + let compositeValidator = CompositeValidator(validators: [ + StructuralValidator(), + ReferenceValidator(), + PathValidator(), + SecurityValidator(), + BestPracticesValidator(), + ]) + + let result = compositeValidator.validate(graph) + + #expect(!result.isValid) + #expect(result.errors.count >= 3) // At least structural, reference, and path errors + #expect(result.warnings.count >= 2) // At least security and best practices warnings + + // Verify we have errors from different validators + let hasStructuralError = result.errors.contains { + if case .duplicateNodeID = $0 { return true } + return false + } + let hasReferenceError = result.errors.contains { + if case .undefinedStageReference = $0 { return true } + return false + } + let hasPathError = result.errors.contains { + if case .emptyDestinationPath = $0 { return true } + return false + } + + #expect(hasStructuralError) + #expect(hasReferenceError) + #expect(hasPathError) + } + + // MARK: - StandardValidator Tests + + @Test func standardValidatorIncludesAllValidators() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let stage = BuildStage( + name: "test", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: ExecOperation(command: .shell("echo 'test'")), + dependencies: [] + ) + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let validator = StandardValidator() + let result = validator.validate(graph) + + #expect(result.isValid) + #expect(result.errors.isEmpty) + + // StandardValidator should include all built-in validators + // This is tested implicitly by verifying it produces the same comprehensive results + } + + // MARK: - ValidationResult Tests + + @Test func validationResultCombining() throws { + let result1 = ValidationResult( + errors: [.duplicateNodeID(id: UUID(), location: .stage(name: "test1"))], + warnings: [.forwardStageReferenceByName(name: "stage2", location: .stage(name: "test1"))] + ) + + let result2 = ValidationResult( + errors: [.missingDependency(dependencyID: UUID(), location: .stage(name: "test2"))], + warnings: [.privilegedExecution(location: .stage(name: "test2"))] + ) + + let combined = ValidationResult.combine([result1, result2]) + + #expect(combined.errors.count == 2) + #expect(combined.warnings.count == 2) + #expect(!combined.isValid) + } + + @Test func validationResultEmpty() throws { + let result = ValidationResult() + + #expect(result.isValid) + #expect(result.errors.isEmpty) + #expect(result.warnings.isEmpty) + } + + // MARK: - Validation Error and Warning Messages Tests + + @Test func validationErrorMessages() throws { + let testId = UUID() + let errors: [ValidationError] = [ + .duplicateNodeID(id: testId, location: .stage(name: "test")), + .cyclicDependency(location: .stage(name: "test")), + .missingDependency(dependencyID: testId, location: .stage(name: "test")), + .undefinedStageReference(name: "missing", location: .stage(name: "test")), + .stageIndexOutOfBounds(index: 99, location: .stage(name: "test")), + .invalidPreviousReference(location: .stage(name: "test")), + .emptyDestinationPath(location: .stage(name: "test")), + .absoluteContextPath(path: "/absolute", location: .stage(name: "test")), + .emptyMountTarget(location: .stage(name: "test")), + ] + + for error in errors { + let description = error.errorDescription + #expect(description != nil, "Error should have description: \(error)") + #expect(!description!.isEmpty, "Error description should not be empty: \(error)") + } + } + + @Test func validationWarningMessages() throws { + let warnings: [ValidationWarning] = [ + .forwardStageReferenceByName(name: "future", location: .stage(name: "test")), + .forwardStageReferenceByIndex(index: 5, location: .stage(name: "test")), + .pathContainsDotDot(path: "../outside", location: .stage(name: "test")), + .privilegedExecution(location: .stage(name: "test")), + .runningAsRoot(location: .stage(name: "test")), + .readWriteSecretMount(location: .stage(name: "test")), + .aptGetUpdateWithoutInstall(location: .stage(name: "test")), + .missingHealthcheck(location: .stage(name: "test")), + ] + + for warning in warnings { + let message = warning.message + #expect(!message.isEmpty, "Warning should have message: \(warning)") + + let suggestion = warning.suggestion + #expect(suggestion != nil, "Warning should have suggestion: \(warning)") + #expect(!suggestion!.isEmpty, "Warning suggestion should not be empty: \(warning)") + } + } + + // MARK: - Performance Tests + + @Test func validationPerformanceWithLargeGraph() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + // Create a large graph with many stages and nodes + var stages: [BuildStage] = [] + + for stageIndex in 0..<10 { + var nodes: [BuildNode] = [] + + // Create 20 nodes per stage + for nodeIndex in 0..<20 { + let operation = ExecOperation(command: .shell("echo 'stage \(stageIndex) node \(nodeIndex)'")) + let node = BuildNode(operation: operation, dependencies: []) + nodes.append(node) + } + + let stage = BuildStage( + name: "stage\(stageIndex)", + base: ImageOperation(source: .registry(alpineRef)), + nodes: nodes + ) + stages.append(stage) + } + + let graph = try BuildGraph(stages: stages) + let validator = StandardValidator() + + // Measure validation time + let startTime = Date() + let result = validator.validate(graph) + let duration = Date().timeIntervalSince(startTime) + + #expect(result.isValid) + #expect(duration < 1.0, "Validation should complete quickly for large graphs (took \(duration)s)") + } + + // MARK: - Custom Validator Tests + + struct CustomSecurityValidator: BuildValidator { + func validate(_ graph: BuildGraph) -> ValidationResult { + var warnings: [ValidationWarning] = [] + + // Custom rule: warn if any stage downloads from HTTP + for stage in graph.stages { + for node in stage.nodes { + if let exec = node.operation as? ExecOperation { + if case .shell(let cmd) = exec.command, + cmd.contains("http://") + { + warnings.append(.privilegedExecution(location: .stage(name: stage.name))) + } + } + } + } + + return ValidationResult(warnings: warnings) + } + } + + @Test func customValidatorExtension() throws { + guard let alpineRef = ImageReference(parsing: "alpine") else { + Issue.record("Failed to parse image reference") + return + } + + let stage = BuildStage( + name: "insecure", + base: ImageOperation(source: .registry(alpineRef)), + nodes: [ + BuildNode( + operation: ExecOperation(command: .shell("curl http://example.com/script.sh | sh")), + dependencies: [] + ) + ] + ) + + let graph = try BuildGraph(stages: [stage]) + let customValidator = CustomSecurityValidator() + let result = customValidator.validate(graph) + + #expect(result.isValid) // No errors, just warnings + #expect(!result.warnings.isEmpty, "Custom validator should detect HTTP usage") + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildParserTests/ParserTests.swift b/Tests/NativeBuilderTests/ContainerBuildParserTests/ParserTests.swift new file mode 100644 index 00000000..cedcd3df --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildParserTests/ParserTests.swift @@ -0,0 +1,658 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import Foundation +import Testing + +@testable import ContainerBuildParser + +@Suite class ParserTest { + @Test func testSimpleDockerfile() throws { + let imageRef = ImageReference(parsing: "alpine:latest") + #expect(imageRef != nil, "Failed to parse image reference") + let dockerfile = + #""" + FROM alpine:latest AS build + """# + let parser = DockerfileParser() + let actualGraph = try parser.parse(dockerfile) + + // check the image reference + #expect(!actualGraph.stages.isEmpty) + #expect(actualGraph.stages.count == 1, "expected 1 stage, instead got \(actualGraph.stages.count)") + #expect(actualGraph.stages[0].name == "build", "expected stage name build, got \(actualGraph.stages[0].name)") + } + + @Test func testSimpleDockerfileLowercase() throws { + let imageRef = ImageReference(parsing: "alpine:latest") + #expect(imageRef != nil, "Failed to parse image reference") + let dockerfile = + #""" + from alpine:latest as base + """# + let parser = DockerfileParser() + let actualGraph = try parser.parse(dockerfile) + + // check the image reference + #expect(!actualGraph.stages.isEmpty) + #expect(actualGraph.stages.count == 1, "expected 1 stage, instead got \(actualGraph.stages.count)") + #expect(actualGraph.stages[0].name == "base", "expected stage name build, got \(actualGraph.stages[0].name)") + } + + @Test func testDockerfileWithContinuation() throws { + let imageRef = ImageReference(parsing: "alpine:latest") + #expect(imageRef != nil, "Failed to parse image reference") + let dockerfile = + #""" + FROM alpine:latest \ + AS \ + build + """# + let parser = DockerfileParser() + let actualGraph = try parser.parse(dockerfile) + + // check the image reference + #expect(!actualGraph.stages.isEmpty) + #expect(actualGraph.stages.count == 1, "expected 1 stage, instead got \(actualGraph.stages.count)") + #expect(actualGraph.stages[0].name == "build", "expected stage name build, got \(actualGraph.stages[0].name)") + } + + static let invalidDockerfileFROM: [String] = [ + #""" + FROM alpine:latest build + """#, + #"FROM alpine:latest build"#, + #"FROM alpine:latest AS"#, + #"FROM AS alpine:latest"#, + #"FROM "" AS build"#, + #"FROM"#, + #""" + FROM alpine:latest \ + build + """#, + ] + + @Test("Invalid FROM instruction throws ParseError", arguments: invalidDockerfileFROM) + func invalidFromDockerfile(_ dockerfile: String) throws { + let parser = DockerfileParser() + #expect(throws: ParseError.self) { + try parser.parse(dockerfile) + } + } +} + +// tests for parsing RUN mount options +extension ParserTest { + struct RunMountTestCase { + let rawMount: String + let expectedRunMount: RunMount? + + init(rawMount: String, expectedRunMount: RunMount? = nil) { + self.rawMount = rawMount + self.expectedRunMount = expectedRunMount + } + } + + static let runMountTestCases = [ + // bind + // basic bind mount with different target option names + RunMountTestCase( + rawMount: "type=bind,dst=/container/dst", + expectedRunMount: RunMount(type: .bind, source: "/", target: "/container/dst", options: RunMountOptions(readonly: true)) + ), + RunMountTestCase( + rawMount: "type=bind,target=/container/target", + expectedRunMount: RunMount(type: .bind, source: "/", target: "/container/target", options: RunMountOptions(readonly: true)) + ), + RunMountTestCase( + rawMount: "type=bind,destination=/container/destination", + expectedRunMount: RunMount(type: .bind, source: "/", target: "/container/destination", options: RunMountOptions(readonly: true)) + ), + // defaults to bind type if none provided + RunMountTestCase( + rawMount: "dst=/container/dst", + expectedRunMount: RunMount(type: .bind, source: "/", target: "/container/dst", options: RunMountOptions(readonly: true)) + ), + // with source + RunMountTestCase( + rawMount: "dst=/container/dst,source=/source", + expectedRunMount: RunMount(type: .bind, source: "/source", target: "/container/dst", options: RunMountOptions(readonly: true)) + ), + // with from + RunMountTestCase( + rawMount: "dst=/container/dst,from=earlierstage", + expectedRunMount: RunMount(type: .bind, source: "/", from: "earlierstage", target: "/container/dst", options: RunMountOptions(readonly: true)) + ), + // with readwrite explicitly set + RunMountTestCase( + rawMount: "type=bind,dst=/container/dst,readwrite=true", + expectedRunMount: RunMount(type: .bind, source: "/", target: "/container/dst", options: RunMountOptions(readonly: false)) + ), + RunMountTestCase( + rawMount: "type=bind,dst=/container/dst,rw=true", + expectedRunMount: RunMount(type: .bind, source: "/", target: "/container/dst", options: RunMountOptions(readonly: false)) + ), + RunMountTestCase( + rawMount: "type=bind,dst=/container/dst,readwrite=false", + expectedRunMount: RunMount(type: .bind, source: "/", target: "/container/dst", options: RunMountOptions(readonly: true)) + ), + RunMountTestCase( + rawMount: "type=bind,dst=/container/dst,readwrite=false", + expectedRunMount: RunMount(type: .bind, source: "/", target: "/container/dst", options: RunMountOptions(readonly: true)) + ), + + /* Cache cases */ + // minimal valid cache + RunMountTestCase( + rawMount: "type=cache,target=/mycache", + expectedRunMount: RunMount( + type: .cache, + source: "/", + from: "", + id: "/mycache", + target: "/mycache", + options: RunMountOptions( + readonly: false, + uid: 0, + gid: 0, + mode: 0755, + sharing: .shared + ) + ) + ), + // normal with all options + RunMountTestCase( + rawMount: "type=cache,id=0987087,target=/target,readonly=false,sharing=private,from=build,source=/source,mode=0,uid=0,gid=0", + expectedRunMount: RunMount( + type: .cache, + source: "/source", + from: "build", + id: "0987087", + target: "/target", + options: RunMountOptions( + readonly: false, + uid: 0, + gid: 0, + mode: 0, + sharing: .private + ) + ) + ), + // cache with sharing + RunMountTestCase( + rawMount: "type=cache,target=/mycache,sharing=shared", + expectedRunMount: RunMount( + type: .cache, + source: "/", + from: "", + id: "/mycache", + target: "/mycache", + options: RunMountOptions( + readonly: false, + uid: 0, + gid: 0, + mode: 0755, + sharing: .shared + ) + ) + ), + RunMountTestCase( + rawMount: "type=cache,target=/mycache,sharing=private", + expectedRunMount: RunMount( + type: .cache, + source: "/", + from: "", + id: "/mycache", + target: "/mycache", + options: RunMountOptions( + readonly: false, + uid: 0, + gid: 0, + mode: 0755, + sharing: .private + ) + ) + ), + RunMountTestCase( + rawMount: "type=cache,target=/mycache,sharing=locked", + expectedRunMount: RunMount( + type: .cache, + source: "/", + from: "", + id: "/mycache", + target: "/mycache", + options: RunMountOptions( + readonly: false, + uid: 0, + gid: 0, + mode: 0755, + sharing: .locked + ) + ) + ), + + // readonly + RunMountTestCase( + rawMount: "type=cache,target=/mycache,readonly=true", + expectedRunMount: RunMount( + type: .cache, + source: "/", + from: "", + id: "/mycache", + target: "/mycache", + options: RunMountOptions( + readonly: true, + uid: 0, + gid: 0, + mode: 0755, + sharing: .shared + ) + ) + ), + + // cache with id, mode, uid, gid + RunMountTestCase( + rawMount: "type=cache,target=/cache,id=cacheid,mode=0444,uid=1001,gid=1002", + expectedRunMount: RunMount( + type: .cache, + source: "/", + from: "", + id: "cacheid", + target: "/cache", + options: RunMountOptions( + readonly: false, + uid: 1001, + gid: 1002, + mode: 0444, + sharing: .shared + ) + ) + ), + + /* tmpfs cases */ + // minimal tmpfs + RunMountTestCase( + rawMount: "type=tmpfs,target=/tmpfs", + expectedRunMount: RunMount( + type: .tmpfs, + target: "/tmpfs", + options: RunMountOptions(readonly: false) + ) + ), + + // size + RunMountTestCase( + rawMount: "type=tmpfs,target=/tmpfs,size=1000", + expectedRunMount: RunMount( + type: .tmpfs, + target: "/tmpfs", + options: RunMountOptions(readonly: false, size: 1000) + ) + ), + + /* secret cases */ + // minimal secret case + RunMountTestCase( + rawMount: "type=secret,target=/run/secrets/mysecret", + expectedRunMount: RunMount( + type: .secret, + id: "mysecret", + target: "/run/secrets/mysecret", + options: RunMountOptions( + readonly: true, + required: false, + uid: 0, + gid: 0, + mode: 0400, + ) + ) + ), + + // secret with id + RunMountTestCase( + rawMount: "type=secret,id=mysecret,target=/run/secrets/mysecret", + expectedRunMount: RunMount( + type: .secret, + id: "mysecret", + target: "/run/secrets/mysecret", + options: RunMountOptions( + readonly: true, + required: false, + uid: 0, + gid: 0, + mode: 0400, + ) + ) + ), + + // secret using env + RunMountTestCase( + rawMount: "type=secret,id=mysecret,target=/run/secrets/mysecret,env=TEST", + expectedRunMount: RunMount( + type: .secret, + id: "mysecret", + env: "TEST", + target: "/run/secrets/mysecret", + options: RunMountOptions( + readonly: true, + required: false, + uid: 0, + gid: 0, + mode: 0400, + ) + ) + ), + + // secret with required, uid/gid/mode + RunMountTestCase( + rawMount: "type=secret,id=mysecret,target=/run/secrets/mysecret,required=true,uid=1000,gid=1001,mode=0400", + expectedRunMount: RunMount( + type: .secret, + id: "mysecret", + target: "/run/secrets/mysecret", + options: RunMountOptions( + readonly: true, + required: true, + uid: 1000, + gid: 1001, + mode: 0400 + ) + ) + ), + + /* ssh cases */ + // minimal ssh mount + RunMountTestCase( + rawMount: "type=ssh", + expectedRunMount: RunMount( + type: .ssh, + id: "default", + target: "/run/buildkit/ssh_agent", + options: RunMountOptions( + readonly: true, + required: false, + uid: 0, + gid: 0, + mode: 0600 + ) + ) + ), + + // ssh with id + RunMountTestCase( + rawMount: "type=ssh,id=myssh,target=/run/ssh", + expectedRunMount: RunMount( + type: .ssh, + id: "myssh", + target: "/run/ssh", + options: RunMountOptions( + readonly: true, + required: false, + uid: 0, + gid: 0, + mode: 0600 + ) + ) + ), + + // ssh with required and id + RunMountTestCase( + rawMount: "type=ssh,id=deploykey,target=/run/ssh,required=true", + expectedRunMount: RunMount( + type: .ssh, + id: "deploykey", + target: "/run/ssh", + options: RunMountOptions( + readonly: true, + required: true, + uid: 0, + gid: 0, + mode: 0600 + ) + ) + ), + ] + + @Test("Run mounts are parsed correctly", arguments: runMountTestCases) + func runParseMount(_ testCase: RunMountTestCase) throws { + let actual = try RunInstruction.parseMount(testCase.rawMount) + #expect(actual == testCase.expectedRunMount) + } + + static let invalidRunMounts: [String] = [ + /* Common cases */ + // missing or mispelled + "type=", + "tyep=bind,target=/container", + "type=bind,targte=/container", + + // duplicate keys + "type=bind,target=/one,target=/two", + "type=bind,dst=/one,destination=/two", + "type=bind,readwrite=true,readwrite=false", + + /* Bind cases */ + // missing or empty values + "type=bind", + "type=bind,target=", + "type=bind,destination", + + // invalid readwrite + "type=bind,target=/target,rw=Not false", + "type=bind,destination=/destination,rw=Totally false", + "type=bind,target=/target,readwrite=yes", + "type=bind,destination=/destination,readwrite=0", + + // malformed key-value format + "type=bind,target=/container,readwrite", + "type=bind,target=/container,readwrite==true", + "type=bind,target=/container,=true", + "type=bind,=true,target=/container", + + // uses options that bind does not support + "type=bind,destination=/destination,mode=0", + "type=bind,target=/destination,gid=0", + "type=bind,dst=/destination,uid=0", + "type=bind,destination=/destination,sharing=private", + "type=bind,target=/destination,required=true", + "type=bind,dst=/destination,size=10", + "type=bind,target=/destination,id=780707", + "type=bind,destination=/destination,env=TEST", + + // uses options that do not exist + "type=bind,target=/target,foo=bar", + "type=bind,target=/target,readwrite=true,invalid-key=value", + + /* Cache cases */ + // missing required target + "type=cache", + "type=cache,target=", + "type=cache,destination", + + // invalid readonly + "type=cache,target=/cache,readonly=", + "type=cache,target=/cache,readonly=Totally", + "type=cache,target=/cache,ro=Forsure", + + // invalid sharing type + "type=cache,target=/cache,sharing=cache", + "type=cache,target=/cache,sharing=none", + + // invalid mode,uid, or gid + "type=cache,target=/cache,mode=-0777", + "type=cache,target=/cache,uid=-1001", + "type=cache,target=/cache,gid=-100", + + // unsupported options + "type=cache,target=/cache,env=TEST", + "type=cache,target=/cache,required=true", + + /* tmpfs cases */ + // tmpfs missing target + "type=tmpfs", + "type=tmpfs,target=", + "type=tmpfs,dst", + + // invalid size + "type=tmpfs,target=/tmpfs,size=", + "type=tmpfs,target=/tmpfs,size=abc", + "type=tmpfs,target=/tmpfs,size=-100", + "type=tmpfs,target=/tmpfs,size=1.5", + + // unsupported options + "type=tmpfs,target=/tmpfs,mode=0755", + "type=tmpfs,target=/tmpfs,uid=0", + "type=tmpfs,target=/tmpfs,gid=0", + "type=tmpfs,target=/tmpfs,id=myid", + "type=tmpfs,target=/tmpfs,from=build", + "type=tmpfs,target=/tmpfs,env=TEST", + "type=tmpfs,target=/tmpfs,required=true", + "type=tmpfs,target=/tmpfs,sharing=private", + + /* secret cases */ + // invalid mode, uid, gid + "type=secret,target=/secret,mode=-0777", + "type=secret,target=/secret,uid=-1001", + "type=secret,target=/secret,gid=-100", + + // missing a target, env, AND id + "type=secret,readonly=true", + + // env set but no target or id + "type=secret,env=TEST,readonly=true", + + // unsupported options + "type=secret,target=/secret,from=build", + "type=secret,target=/secret,source=/", + "type=secret,target=/secret,sharing=private", + + /* ssh cases */ + // invalid mode,uid,gid + "type=ssh,mode=-0755", + "type=ssh,uid=-079", + "type=ssh,gid=", + + // unsupported options + "type=ssh,target=/ssh,from=build", + "type=ssh,target=/ssh,source=/", + "type=ssh,target=/ssh,env=SSH_TEST", + "type=ssh,target=/ssh,sharing=private", + ] + + @Test("Invalid run mount configuration throws error", arguments: invalidRunMounts) + func testInvalidRunMounts(_ testCase: String) throws { + #expect(throws: ParseError.self) { + let _ = try RunInstruction.parseMount(testCase) + } + } + + struct RunNetworkTest { + let rawNetwork: String? + let expectedNetwork: NetworkMode? + + init(rawNetwork: String?, expectedNetwork: NetworkMode? = nil) { + self.rawNetwork = rawNetwork + self.expectedNetwork = expectedNetwork + } + } + + static let runNetworkTests: [RunNetworkTest] = [ + RunNetworkTest(rawNetwork: "default", expectedNetwork: .default), + RunNetworkTest(rawNetwork: "none", expectedNetwork: NetworkMode.none), + RunNetworkTest(rawNetwork: "host", expectedNetwork: .host), + RunNetworkTest(rawNetwork: nil, expectedNetwork: .default), + ] + + @Test("Successful run network parsing", arguments: runNetworkTests) + func runNetworkParseTest(_ testCase: RunNetworkTest) throws { + let actual = try RunInstruction.parseNetworkMode(mode: testCase.rawNetwork) + #expect(actual == testCase.expectedNetwork) + } + + @Test func invalidNetworkParse() throws { + let invalidMode = "fake" + #expect(throws: ParseError.self) { + let _ = try RunInstruction.parseNetworkMode(mode: invalidMode) + } + } + + struct CopyOwnershipTest { + let rawOwnership: String? + let expectedOwnership: Ownership? + } + + static let copyOwnershipTests = [ + CopyOwnershipTest( + rawOwnership: "55:mygroup", + expectedOwnership: Ownership(user: .numeric(id: 55), group: .named(id: "mygroup")) + ), + CopyOwnershipTest( + rawOwnership: "bin", + expectedOwnership: Ownership(user: .named(id: "bin"), group: nil) + ), + CopyOwnershipTest( + rawOwnership: "1", + expectedOwnership: Ownership(user: .numeric(id: 1), group: nil) + ), + CopyOwnershipTest( + rawOwnership: "10:11", + expectedOwnership: Ownership(user: .numeric(id: 10), group: .numeric(id: 11)) + ), + CopyOwnershipTest( + rawOwnership: "myuser:mygroup", + expectedOwnership: Ownership(user: .named(id: "myuser"), group: .named(id: "mygroup")) + ), + CopyOwnershipTest( + rawOwnership: "", + expectedOwnership: Ownership(user: .numeric(id: 0), group: .numeric(id: 0)) + ), + CopyOwnershipTest( + rawOwnership: nil, + expectedOwnership: Ownership(user: .numeric(id: 0), group: .numeric(id: 0)) + ), + CopyOwnershipTest( + rawOwnership: ":mygroup", + expectedOwnership: Ownership(user: nil, group: .named(id: "mygroup")) + ), + ] + + @Test("Expected parsing of chown options", arguments: copyOwnershipTests) + func testCopyParseOwnership(_ testcase: CopyOwnershipTest) throws { + let actual = try CopyInstruction.parseOwnership(input: testcase.rawOwnership) + #expect(actual == testcase.expectedOwnership) + } + + @Test func testCopyParseOwnershipInvalid() throws { + let rawInput = "myuser:mygroup:extra" + #expect(throws: ParseError.self) { + let _ = try CopyInstruction.parseOwnership(input: rawInput) + } + } + + @Test func testCopyParsePermissions() throws { + let rawPermission = "777" + let actual = try CopyInstruction.parsePermissions(input: rawPermission) + #expect(actual == .mode(777)) + } + + @Test func testCopyParsePermissionsInvalid() throws { + let rawPermission = "u+x" + #expect(throws: ParseError.self) { + let _ = try CopyInstruction.parsePermissions(input: rawPermission) + } + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildParserTests/TokenizerTests.swift b/Tests/NativeBuilderTests/ContainerBuildParserTests/TokenizerTests.swift new file mode 100644 index 00000000..00911755 --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildParserTests/TokenizerTests.swift @@ -0,0 +1,444 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import Testing + +@testable import ContainerBuildParser + +@Suite class TokenizerTest { + struct tokenizerTestInput { + let input: String + let expectedTokens: [Token] + } + + let tokenizerTestCases: [tokenizerTestInput] = [ + tokenizerTestInput( + input: "FROM alpine AS build", + expectedTokens: [ + .stringLiteral("FROM"), + .stringLiteral("alpine"), + .stringLiteral("AS"), + .stringLiteral("build"), + ] + ), + tokenizerTestInput( + input: "FROM alpine", + expectedTokens: [ + .stringLiteral("FROM"), + .stringLiteral("alpine"), + ] + ), + tokenizerTestInput( + input: "RUN --mount=type=cache /app", + expectedTokens: [ + .stringLiteral("RUN"), + .option("--mount", "type=cache"), + .stringLiteral("/app"), + ] + ), + tokenizerTestInput( + input: "RUN --network=default /app", + expectedTokens: [ + .stringLiteral("RUN"), + .option("--network", "default"), + .stringLiteral("/app"), + ] + ), + tokenizerTestInput( + input: "RUN --mount=type=bind,target=/target --network=host build.sh", + expectedTokens: [ + .stringLiteral("RUN"), + .option("--mount", "type=bind,target=/target"), + .option("--network", "host"), + .stringLiteral("build.sh"), + ] + ), + tokenizerTestInput( + input: "RUN --mount type=cache /app", + expectedTokens: [ + .stringLiteral("RUN"), + .option("--mount", "type=cache"), + .stringLiteral("/app"), + ] + ), + tokenizerTestInput( + input: + """ + RUN --mount=type=cache build.sh --input hello + """, + expectedTokens: [ + .stringLiteral("RUN"), + .option("--mount", "type=cache"), + .stringLiteral("build.sh"), + .option("--input", "hello"), + ] + ), + tokenizerTestInput( + input: + #""" + RUN --mount=type=cache ["build.sh", "--input", "hello"] + """#, + expectedTokens: [ + .stringLiteral("RUN"), + .option("--mount", "type=cache"), + .stringList(["build.sh", "--input", "hello"]), + ] + ), + tokenizerTestInput( + input: + #""" + RUN --mount=type=cache [ "build.sh", "--input", "hello" ] + """#, + expectedTokens: [ + .stringLiteral("RUN"), + .option("--mount", "type=cache"), + .stringList(["build.sh", "--input", "hello"]), + ] + ), + tokenizerTestInput( + input: + #""" + RUN --mount=type=cache "build.sh --input hello" + """#, + expectedTokens: [ + .stringLiteral("RUN"), + .option("--mount", "type=cache"), + .stringLiteral("build.sh --input hello"), + ] + ), + tokenizerTestInput( + input: + #""" + RUN --mount=type=cache "build.sh --input hello" + # this is a full line comment + """#, + expectedTokens: [ + .stringLiteral("RUN"), + .option("--mount", "type=cache"), + .stringLiteral("build.sh --input hello"), + ] + ), + tokenizerTestInput( + input: + #""" + RUN --mount=type=cache "build.sh --input hello" # this is an end line comment + """#, + expectedTokens: [ + .stringLiteral("RUN"), + .option("--mount", "type=cache"), + .stringLiteral("build.sh --input hello"), + ] + ), + tokenizerTestInput( + input: "COPY --from=alpine src /dest", + expectedTokens: [ + .stringLiteral("COPY"), + .option("--from", "alpine"), + .stringLiteral("src"), + .stringLiteral("/dest"), + ] + ), + + tokenizerTestInput( + input: "COPY --from=alpine src src1 src2 src3 /dest", + expectedTokens: [ + .stringLiteral("COPY"), + .option("--from", "alpine"), + .stringLiteral("src"), + .stringLiteral("src1"), + .stringLiteral("src2"), + .stringLiteral("src3"), + .stringLiteral("/dest"), + ] + ), + tokenizerTestInput( + input: "COPY --chown=10:11 src /dest", + expectedTokens: [ + .stringLiteral("COPY"), + .option("--chown", "10:11"), + .stringLiteral("src"), + .stringLiteral("/dest"), + ] + ), + tokenizerTestInput( + input: "COPY --chown=bin stuff.txt /stuffdest/", + expectedTokens: [ + .stringLiteral("COPY"), + .option("--chown", "bin"), + .stringLiteral("stuff.txt"), + .stringLiteral("/stuffdest/"), + ] + ), + tokenizerTestInput( + input: "COPY --chown=1 source /destination", + expectedTokens: [ + .stringLiteral("COPY"), + .option("--chown", "1"), + .stringLiteral("source"), + .stringLiteral("/destination"), + ] + ), + tokenizerTestInput( + input: "COPY --chmod=440 src /dest/", + expectedTokens: [ + .stringLiteral("COPY"), + .option("--chmod", "440"), + .stringLiteral("src"), + .stringLiteral("/dest/"), + ] + ), + tokenizerTestInput( + input: "COPY --link=false src /dest/", + expectedTokens: [ + .stringLiteral("COPY"), + .option("--link", "false"), + .stringLiteral("src"), + .stringLiteral("/dest/"), + ] + ), + ] + + @Test func testTokenization() throws { + for testCase: tokenizerTestInput in tokenizerTestCases { + var tokenizer = DockerfileTokenizer(testCase.input) + let tokens = try tokenizer.getTokens() + #expect(!tokens.isEmpty) + #expect(TokenizerTest.isEqual(actual: tokens, expected: testCase.expectedTokens)) + } + } + + static private func isEqual(actual: [Token], expected: [Token]) -> Bool { + if actual.count != expected.count { + return false + } + var index = 0 + while index < actual.count { + if actual[index] != expected[index] { + return false + } + index += 1 + } + return true + } + + struct TokenTest { + let tokens: [Token] + let expectedInstruction: DockerInstruction + } + + @Test func tokenTranslationFrom() throws { + let fromTokenTestInputs: [TokenTest] = [ + TokenTest( + tokens: [ + .stringLiteral("FROM"), + .stringLiteral("alpine"), + ], + expectedInstruction: try FromInstruction(image: "alpine") + ), + TokenTest( + tokens: [ + .stringLiteral("FROM"), + .stringLiteral("alpine"), + .stringLiteral("AS"), + .stringLiteral("build"), + ], + expectedInstruction: try FromInstruction(image: "alpine", stageName: "build") + ), + TokenTest( + tokens: [ + .stringLiteral("FROM"), + .option("--platform", "linux/arm64"), + .stringLiteral("alpine"), + ], + expectedInstruction: try FromInstruction(image: "alpine", platform: "linux/arm64") + ), + ] + + for testCase in fromTokenTestInputs { + let buildParser = DockerfileParser() + let actualInstruction = try buildParser.tokensToFromInstruction(tokens: testCase.tokens) + guard let expected = testCase.expectedInstruction as? FromInstruction else { + Issue.record("unexpected instruction type \(testCase.expectedInstruction)") + return + } + #expect(actualInstruction == expected) + } + } + + @Test func testTokensToRunWithShellCommand() throws { + let tokens: [Token] = [ + .stringLiteral("RUN"), + .option("--mount", "type=cache,target=/cache"), + .stringLiteral("build.sh --input hello"), + ] + + let parser = DockerfileParser() + let actual = try parser.tokensToRunInstruction(tokens: tokens) + + #expect(actual.shell) + #expect(actual.command == "build.sh --input hello") + } + + @Test func testTokensToRunWithoutShell() throws { + let command = ["build.sh", "--input", "hello"] + let tokens: [Token] = [ + .stringLiteral("RUN"), + .option("--mount", "type=cache,target=/mytarget"), + .stringList(command), + ] + + let parser = DockerfileParser() + let actual = try parser.tokensToRunInstruction(tokens: tokens) + + #expect(actual.shell == false) + #expect(actual.command == command.joined(separator: " ")) + } + + static let extraTokensTests: [[Token]] = [ + [ + .stringLiteral("RUN"), + .option("--mount", "type=tmpfs,size=1000"), + .stringList(["build.sh", "--input", "hello"]), + .stringLiteral("extra"), + ], + [ + .stringLiteral("RUN"), + .option("--mount", "type=bind,target=/target"), + .stringLiteral("build.sh"), + .stringLiteral("--input"), + .stringLiteral("hello"), + .stringList(["extra"]), + ], + [ + .stringLiteral("RUN"), + .option("--mount", "type=bind,target=/target"), + .stringLiteral("build.sh"), + .option("key", "value"), + ], + ] + + @Test("Parsing to run instruction fails when there's extra tokens", arguments: extraTokensTests) + func testTokensToRunExtraTokens(tokens: [Token]) throws { + #expect(throws: ParseError.self) { + let parser = DockerfileParser() + let _ = try parser.tokensToRunInstruction(tokens: tokens) + } + } + + @Test func testTokensToCopyInstruction() throws { + let copyTokenTests = [ + TokenTest( + tokens: [ + .stringLiteral("COPY"), + .option("--link", "false"), + .stringLiteral("src"), + .stringLiteral("/dest/"), + ], + expectedInstruction: try CopyInstruction( + sources: ["src"], + destination: "/dest/", + ) + ), + TokenTest( + tokens: [ + .stringLiteral("COPY"), + .option("--chmod", "440"), + .stringLiteral("src"), + .stringLiteral("/dest"), + ], + expectedInstruction: try CopyInstruction( + sources: ["src"], + destination: "/dest", + permissions: .mode(440) + ) + ), + TokenTest( + tokens: [ + .stringLiteral("COPY"), + .option("--chown", "11:mygroup"), + .stringLiteral("source"), + .stringLiteral("destination"), + ], + expectedInstruction: try CopyInstruction( + sources: ["source"], + destination: "destination", + ownership: Ownership(user: .numeric(id: 11), group: .named(id: "mygroup")) + ) + ), + TokenTest( + tokens: [ + .stringLiteral("COPY"), + .option("--from", "alpine"), + .stringLiteral("src"), + .stringLiteral("src1"), + .stringLiteral("src2"), + .stringLiteral("src3"), + .stringLiteral("/dest"), + ], + expectedInstruction: try CopyInstruction( + sources: ["src", "src1", "src2", "src3"], + destination: "/dest", + from: "alpine", + ) + ), + TokenTest( + tokens: [ + .stringLiteral("COPY"), + .option("--from", "base"), + .stringLiteral("Source"), + .stringLiteral("Dest"), + ], + expectedInstruction: try CopyInstruction( + sources: ["Source"], + destination: "Dest", + from: "base", + ) + ), + ] + + for test in copyTokenTests { + let parser = DockerfileParser() + let actual = try parser.tokensToCopyInstruction(tokens: test.tokens) + guard let expected = test.expectedInstruction as? CopyInstruction else { + Issue.record("unexpected instruction type \(test.expectedInstruction)") + return + } + #expect(actual == expected) + } + } + + static let invalidCopyTokens: [[Token]] = [ + [ + // no destination + .stringLiteral("COPY"), + .stringLiteral("Source"), + ], + [ + // no sources + .stringLiteral("COPY"), + .option("--from", "alpine"), + ], + ] + + @Test("Invalid copy tokens throw an error", arguments: invalidCopyTokens) + func testInvalidCopyTokens(_ tokens: [Token]) throws { + let parser = DockerfileParser() + #expect(throws: ParseError.self) { + let _ = try parser.tokensToCopyInstruction(tokens: tokens) + } + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildParserTests/VisitorTests.swift b/Tests/NativeBuilderTests/ContainerBuildParserTests/VisitorTests.swift new file mode 100644 index 00000000..571f23e8 --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildParserTests/VisitorTests.swift @@ -0,0 +1,121 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerizationOCI +import Testing + +@testable import ContainerBuildParser + +@Suite class VisitorTest { + @Test func simpleVisitFrom() throws { + let imageName = "alpine" + let expectedImageRef = ImageReference(parsing: imageName) + let stageName = "build" + let platformString = "linux/arm64" + let expectedPlatform = try Platform(from: platformString) + + let from = try FromInstruction(image: imageName, platform: platformString, stageName: stageName) + let visitor = DockerInstructionVisitor() + try visitor.visit(from) + + let graph = try visitor.graphBuilder.build() + #expect(graph.stages.count == 1) + let stage = graph.stages[0] + #expect(stage.name == stageName) + #expect(stage.platform! == expectedPlatform) + #expect(stage.base.source == .registry(expectedImageRef!)) + } + + @Test func simpleVisitRun() throws { + let from = try FromInstruction(image: "scratch") + let command = ["sh", "-c", "top"] + let network = "default" + let run = try RunInstruction( + command: command, + shell: false, + rawMounts: [], + network: network + ) + + let visitor = DockerInstructionVisitor() + try visitor.visit(from) + try visitor.visit(run) + + let graph = try visitor.graphBuilder.build() + + #expect(graph.stages.count == 1) + let stage = graph.stages[0] + + #expect(stage.nodes.count == 1) + let node = stage.nodes[0] + + #expect(node.operation is ExecOperation) + + guard let exec = node.operation as? ExecOperation else { + Issue.record("expected ExecOperation, instead got \(node.operation)") + return + } + + #expect(exec.command.arguments == command) + #expect(exec.mounts.isEmpty) + #expect(exec.network == .default) + } + + @Test func simpleVisitCopy() throws { + let from = try FromInstruction(image: "scratch") + let sources = ["src", "src1", "src2"] + let dest = "/dest" + let copy = try CopyInstruction( + sources: sources, + destination: dest, + from: nil, + ownership: "10:10", + permissions: "777" + ) + + let visitor = DockerInstructionVisitor() + try visitor.visit(from) + try visitor.visit(copy) + + let graph = try visitor.graphBuilder.build() + + #expect(graph.stages.count == 1) + let stage = graph.stages[0] + + #expect(stage.nodes.count == 1) + let node = stage.nodes[0] + + guard let copyNode = node.operation as? FilesystemOperation else { + Issue.record("expected FilesystemOperation, instead got \(node.operation)") + return + } + + #expect(copyNode.action == .copy) + + let expectedSource = ContextSource( + name: "default", + paths: sources) + #expect(copyNode.source == .context(expectedSource)) + #expect(copyNode.destination == dest) + + let expectedOwnership = Ownership(user: .numeric(id: 10), group: .numeric(id: 10)) + #expect(copyNode.fileMetadata.ownership == expectedOwnership) + + let expectedPerms: Permissions = .mode(777) + #expect(copyNode.fileMetadata.permissions == expectedPerms) + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildReportingTests/BaseProgressConsumerTests.swift b/Tests/NativeBuilderTests/ContainerBuildReportingTests/BaseProgressConsumerTests.swift new file mode 100644 index 00000000..478f87be --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildReportingTests/BaseProgressConsumerTests.swift @@ -0,0 +1,679 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation +import Testing + +@testable import ContainerBuildReporting + +@Suite +struct BaseProgressConsumerTests { + + // MARK: - Test Configuration + + struct TestConfiguration: Sendable { + let name: String + let bufferSize: Int + + init(name: String = "test", bufferSize: Int = 100) { + self.name = name + self.bufferSize = bufferSize + } + } + + // Test concrete implementation of BaseProgressConsumer + @MainActor + final class TestProgressConsumer: BaseProgressConsumer, @unchecked Sendable { + private var formattedEvents: [String] = [] + + override func formatAndOutput(_ event: BuildEvent) async throws { + formattedEvents.append("Formatted: \(event)") + } + + func getFormattedEvents() -> [String] { + formattedEvents + } + + func clearFormattedEvents() { + formattedEvents.removeAll() + } + } + + // MARK: - Initialization Tests + + @Test("Initialization with configuration") + @MainActor + func initializationWithConfiguration() { + let config = TestConfiguration(name: "test-consumer", bufferSize: 200) + let consumer = TestProgressConsumer(configuration: config) + + #expect(consumer.configuration.name == "test-consumer") + #expect(consumer.configuration.bufferSize == 200) + #expect(consumer.getEvents().isEmpty) + #expect(consumer.getFormattedEvents().isEmpty) + } + + @Test("Initialization with default configuration") + @MainActor + func initializationWithDefaultConfiguration() { + let consumer = TestProgressConsumer(configuration: TestConfiguration()) + + #expect(consumer.configuration.name == "test") + #expect(consumer.configuration.bufferSize == 100) + #expect(consumer.getEvents().isEmpty) + } + + // MARK: - Event Handling Tests + + @Test("Handles build started event") + @MainActor + func handlesBuildStartedEvent() async throws { + let consumer = TestProgressConsumer(configuration: TestConfiguration()) + let event = BuildEvent.buildStarted(totalOperations: 10, stages: 1, timestamp: Date()) + + try await consumer.handle(event) + + let events = consumer.getEvents() + #expect(events.count == 1) + + let statistics = consumer.getStatistics() + #expect(statistics.startTime != nil) + #expect(statistics.totalOperations == 10) + #expect(statistics.endTime == nil) + #expect(statistics.success == nil) + } + + @Test("Handles build completed event") + @MainActor + func handlesBuildCompletedEvent() async throws { + let consumer = TestProgressConsumer(configuration: TestConfiguration()) + let event = BuildEvent.buildCompleted(success: true, timestamp: Date()) + + try await consumer.handle(event) + + let events = consumer.getEvents() + #expect(events.count == 1) + + let statistics = consumer.getStatistics() + #expect(statistics.endTime != nil) + #expect(statistics.success == true) + } + + @Test("Handles stage started event") + @MainActor + func handlesStageStartedEvent() async throws { + let consumer = TestProgressConsumer(configuration: TestConfiguration()) + let event = BuildEvent.stageStarted(stageName: "stage1", timestamp: Date()) + + try await consumer.handle(event) + + let statistics = consumer.getStatistics() + #expect(statistics.totalStages == 1) + #expect(statistics.stageStatistics["stage1"] != nil) + #expect(statistics.stageStatistics["stage1"]?.startTime != nil) + #expect(statistics.stageStatistics["stage1"]?.endTime == nil) + } + + @Test("Handles stage completed event") + @MainActor + func handlesStageCompletedEvent() async throws { + let consumer = TestProgressConsumer(configuration: TestConfiguration()) + + // Start stage first + let startEvent = BuildEvent.stageStarted(stageName: "stage1", timestamp: Date()) + try await consumer.handle(startEvent) + + // Complete stage + let completeEvent = BuildEvent.stageCompleted(stageName: "stage1", timestamp: Date()) + try await consumer.handle(completeEvent) + + let statistics = consumer.getStatistics() + #expect(statistics.totalStages == 1) + #expect(statistics.stageStatistics["stage1"]?.startTime != nil) + #expect(statistics.stageStatistics["stage1"]?.endTime != nil) + } + + @Test("Handles operation started event") + @MainActor + func handlesOperationStartedEvent() async throws { + let consumer = TestProgressConsumer(configuration: TestConfiguration()) + let context = ReportContext(nodeId: UUID(), stageId: "stage1", description: "Operation 1") + let event = BuildEvent.operationStarted(context: context) + + try await consumer.handle(event) + + let statistics = consumer.getStatistics() + #expect(statistics.stageStatistics["stage1"]?.operationCount == 1) + #expect(statistics.stageStatistics["stage1"]?.cacheHits == 0) + #expect(statistics.stageStatistics["stage1"]?.failures == 0) + } + + @Test("Handles operation finished event") + @MainActor + func handlesOperationFinishedEvent() async throws { + let consumer = TestProgressConsumer(configuration: TestConfiguration()) + let event = BuildEvent.operationFinished(context: ReportContext(nodeId: UUID(), description: "Operation 1"), duration: 1.0) + + try await consumer.handle(event) + + let statistics = consumer.getStatistics() + #expect(statistics.executedOperations == 1) + #expect(statistics.failedOperations == 0) + #expect(statistics.cacheHits == 0) + } + + @Test("Handles operation failed event") + @MainActor + func handlesOperationFailedEvent() async throws { + let consumer = TestProgressConsumer(configuration: TestConfiguration()) + let context = ReportContext(nodeId: UUID(), stageId: "stage1", description: "Operation 1") + let error = BuildEventError(type: .executionFailed, description: "Test failure") + let event = BuildEvent.operationFailed(context: context, error: error) + + try await consumer.handle(event) + + let statistics = consumer.getStatistics() + #expect(statistics.failedOperations == 1) + #expect(statistics.executedOperations == 0) + #expect(statistics.stageStatistics["stage1"]?.failures == 1) + } + + @Test("Handles operation cache hit event") + @MainActor + func handlesOperationCacheHitEvent() async throws { + let consumer = TestProgressConsumer(configuration: TestConfiguration()) + let context = ReportContext(nodeId: UUID(), stageId: "stage1", description: "Operation 1") + let event = BuildEvent.operationCacheHit(context: context) + + try await consumer.handle(event) + + let statistics = consumer.getStatistics() + #expect(statistics.cacheHits == 1) + #expect(statistics.stageStatistics["stage1"]?.cacheHits == 1) + } + + @Test("Handles operation progress event") + @MainActor + func handlesOperationProgressEvent() async throws { + let consumer = TestProgressConsumer(configuration: TestConfiguration()) + let event = BuildEvent.operationProgress(context: ReportContext(nodeId: UUID(), description: "Operation 1"), fraction: 0.5) + + try await consumer.handle(event) + + let events = consumer.getEvents() + #expect(events.count == 1) + + // Progress events don't affect statistics + let statistics = consumer.getStatistics() + #expect(statistics.executedOperations == 0) + #expect(statistics.failedOperations == 0) + #expect(statistics.cacheHits == 0) + } + + @Test("Handles operation log event") + @MainActor + func handlesOperationLogEvent() async throws { + let consumer = TestProgressConsumer(configuration: TestConfiguration()) + let event = BuildEvent.operationLog(context: ReportContext(nodeId: UUID(), description: "Operation 1"), message: "Log message") + + try await consumer.handle(event) + + let events = consumer.getEvents() + #expect(events.count == 1) + + // Log events don't affect statistics + let statistics = consumer.getStatistics() + #expect(statistics.executedOperations == 0) + #expect(statistics.failedOperations == 0) + #expect(statistics.cacheHits == 0) + } + + @Test("Handles IR event") + @MainActor + func handlesIREvent() async throws { + let consumer = TestProgressConsumer(configuration: TestConfiguration()) + let event = BuildEvent.irEvent(context: ReportContext(nodeId: UUID(), description: "IR event"), type: .graphStarted) + + try await consumer.handle(event) + + let events = consumer.getEvents() + #expect(events.count == 1) + + // IR events don't affect statistics + let statistics = consumer.getStatistics() + #expect(statistics.executedOperations == 0) + #expect(statistics.failedOperations == 0) + #expect(statistics.cacheHits == 0) + } + + // MARK: - Statistics Accumulation Tests + + @Test("Statistics accumulation with complex sequence") + @MainActor + func statisticsAccumulationWithComplexSequence() async throws { + let consumer = TestProgressConsumer(configuration: TestConfiguration()) + + // Build sequence: start -> 2 stages -> multiple operations -> complete + let events = [ + BuildEvent.buildStarted(totalOperations: 6, stages: 2, timestamp: Date()), + BuildEvent.stageStarted(stageName: "stage1", timestamp: Date()), + BuildEvent.operationStarted(context: ReportContext(nodeId: UUID(), stageId: "stage1", description: "Operation 1")), + BuildEvent.operationFinished(context: ReportContext(nodeId: UUID(), stageId: "stage1", description: "Operation 1"), duration: 1.0), + BuildEvent.operationStarted(context: ReportContext(nodeId: UUID(), stageId: "stage1", description: "Operation 2")), + BuildEvent.operationCacheHit(context: ReportContext(nodeId: UUID(), stageId: "stage1", description: "Operation 2")), + BuildEvent.operationStarted(context: ReportContext(nodeId: UUID(), stageId: "stage1", description: "Operation 3")), + BuildEvent.operationFailed( + context: ReportContext(nodeId: UUID(), stageId: "stage1", description: "Operation 3"), error: BuildEventError(type: .executionFailed, description: "Failed")), + BuildEvent.stageCompleted(stageName: "stage1", timestamp: Date()), + BuildEvent.stageStarted(stageName: "stage2", timestamp: Date()), + BuildEvent.operationStarted(context: ReportContext(nodeId: UUID(), stageId: "stage2", description: "Operation 4")), + BuildEvent.operationFinished(context: ReportContext(nodeId: UUID(), stageId: "stage2", description: "Operation 4"), duration: 1.0), + BuildEvent.operationStarted(context: ReportContext(nodeId: UUID(), stageId: "stage2", description: "Operation 5")), + BuildEvent.operationCacheHit(context: ReportContext(nodeId: UUID(), stageId: "stage2", description: "Operation 5")), + BuildEvent.operationStarted(context: ReportContext(nodeId: UUID(), stageId: "stage2", description: "Operation 6")), + BuildEvent.operationFinished(context: ReportContext(nodeId: UUID(), stageId: "stage2", description: "Operation 6"), duration: 1.0), + BuildEvent.stageCompleted(stageName: "stage2", timestamp: Date()), + BuildEvent.buildCompleted(success: false, timestamp: Date()), + ] + + for event in events { + try await consumer.handle(event) + } + + let statistics = consumer.getStatistics() + + // Verify build-level statistics + #expect(statistics.totalOperations == 6) + #expect(statistics.executedOperations == 3) // op1, op4, op6 + #expect(statistics.cacheHits == 2) // op2, op5 + #expect(statistics.failedOperations == 1) // op3 + #expect(statistics.success == false) + #expect(statistics.totalStages == 2) + #expect(statistics.startTime != nil) + #expect(statistics.endTime != nil) + #expect(statistics.events.count == 18) + + // Verify stage1 statistics + let stage1Stats = statistics.stageStatistics["stage1"] + #expect(stage1Stats != nil) + #expect(stage1Stats?.operationCount == 3) + #expect(stage1Stats?.cacheHits == 1) + #expect(stage1Stats?.failures == 1) + #expect(stage1Stats?.startTime != nil) + #expect(stage1Stats?.endTime != nil) + + // Verify stage2 statistics + let stage2Stats = statistics.stageStatistics["stage2"] + #expect(stage2Stats != nil) + #expect(stage2Stats?.operationCount == 3) + #expect(stage2Stats?.cacheHits == 1) + #expect(stage2Stats?.failures == 0) + #expect(stage2Stats?.startTime != nil) + #expect(stage2Stats?.endTime != nil) + } + + @Test("Statistics with multiple stages") + @MainActor + func statisticsWithMultipleStages() async throws { + let consumer = TestProgressConsumer(configuration: TestConfiguration()) + + // Create multiple stages with different characteristics + let stages = [ + ("stage1", 5, 2, 1), // 5 ops, 2 cache hits, 1 failure + ("stage2", 3, 1, 0), // 3 ops, 1 cache hit, 0 failures + ("stage3", 2, 0, 2), // 2 ops, 0 cache hits, 2 failures + ("stage4", 1, 1, 0), // 1 op, 1 cache hit, 0 failures + ] + + for (stageName, opCount, cacheHits, failures) in stages { + let stageStartEvent = BuildEvent.stageStarted(stageName: stageName, timestamp: Date()) + try await consumer.handle(stageStartEvent) + + // Add operations + for i in 0..?" + let stageStats = StageStatistics(name: name) + + #expect(stageStats.name == name) + } + + @Test("StageStatistics with unicode characters in name") + func testStageStatisticsWithUnicodeCharactersInName() { + let name = "测试阶段-🚀-étape-тест" + let stageStats = StageStatistics(name: name) + + #expect(stageStats.name == name) + } + + @Test("StageStatistics with zero operations") + func testStageStatisticsWithZeroOperations() { + let stageStats = StageStatistics(name: "empty-stage", operationCount: 0, cacheHits: 0, failures: 0) + + #expect(stageStats.operationCount == 0) + #expect(stageStats.cacheHits == 0) + #expect(stageStats.failures == 0) + } + + @Test("StageStatistics with high operation counts") + func testStageStatisticsWithHighOperationCounts() { + let stageStats = StageStatistics(name: "large-stage", operationCount: 10000, cacheHits: 5000, failures: 100) + + #expect(stageStats.operationCount == 10000) + #expect(stageStats.cacheHits == 5000) + #expect(stageStats.failures == 100) + } + + @Test("StageStatistics with all cache hits") + func testStageStatisticsWithAllCacheHits() { + let stageStats = StageStatistics(name: "cached-stage", operationCount: 10, cacheHits: 10, failures: 0) + + #expect(stageStats.operationCount == 10) + #expect(stageStats.cacheHits == 10) + #expect(stageStats.failures == 0) + } + + @Test("StageStatistics with all failures") + func testStageStatisticsWithAllFailures() { + let stageStats = StageStatistics(name: "failed-stage", operationCount: 5, cacheHits: 0, failures: 5) + + #expect(stageStats.operationCount == 5) + #expect(stageStats.cacheHits == 0) + #expect(stageStats.failures == 5) + } + + // MARK: - Edge Cases and Performance Tests + + @Test("BuildStatistics with extremely long duration") + func testBuildStatisticsWithExtremelyLongDuration() { + let startTime = Date(timeIntervalSince1970: 0) + let endTime = Date() + + let stats = BuildStatistics(startTime: startTime, endTime: endTime) + + #expect(stats.duration != nil) + #expect(stats.duration! > 0) + } + + @Test("BuildStatistics with very short duration") + func testBuildStatisticsWithVeryShortDuration() { + let startTime = Date() + let endTime = Date(timeInterval: 0.001, since: startTime) + + let stats = BuildStatistics(startTime: startTime, endTime: endTime) + + #expect(abs(stats.duration! - 0.001) < 0.0001) + } + + @Test("BuildStatistics memory usage with large data set") + func testBuildStatisticsMemoryUsageWithLargeDataSet() { + let eventCount = 100000 + let stageCount = 1000 + + let events = (0.. (FileHandle, FileHandle) { + let pipe = Pipe() + return (pipe.fileHandleForWriting, pipe.fileHandleForReading) + } + + private func readOutputAsString(from readHandle: FileHandle) -> String { + let data = readHandle.readDataToEndOfFile() + return String(data: data, encoding: .utf8) ?? "" + } + + private func parseJSONLines(_ output: String) -> [[String: Any]] { + let lines = output.split(separator: "\n").map(String.init) + return lines.compactMap { line in + guard let data = line.data(using: .utf8), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] + else { + return nil + } + return json + } + } + + // MARK: - Initialization Tests + + @Test("Default initialization sets correct configuration values") + func testDefaultInitialization() { + let consumer = JSONProgressConsumer(configuration: JSONProgressConsumer.Configuration()) + + #expect(!consumer.configuration.prettyPrint) + #expect(consumer.configuration.output == FileHandle.standardOutput) + } + + @Test("Initialization with custom configuration applies settings correctly") + func testInitializationWithCustomConfiguration() { + let (writeHandle, _) = createTestPipe() + let config = JSONProgressConsumer.Configuration( + output: writeHandle, + prettyPrint: true + ) + let consumer = JSONProgressConsumer(configuration: config) + + #expect(consumer.configuration.prettyPrint) + #expect(consumer.configuration.output == writeHandle) + } + + // MARK: - Build Event JSON Output Tests + + @Test("Build started event produces correct JSON output") + func testBuildStartedEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let event = BuildEvent.buildStarted(totalOperations: 5, stages: 2, timestamp: Date()) + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let jsonLines = parseJSONLines(output) + + #expect(jsonLines.count == 1) + let json = jsonLines[0] + #expect(json["type"] as? String == "build_started") + #expect(json["total_operations"] as? Int == 5) + #expect(json["stages"] as? Int == 2) + #expect(json["timestamp"] != nil) + } + + @Test("Build completed event produces correct JSON output") + func testBuildCompletedEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let event = BuildEvent.buildCompleted(success: true, timestamp: Date()) + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let jsonLines = parseJSONLines(output) + + #expect(jsonLines.count == 1) + let json = jsonLines[0] + #expect(json["type"] as? String == "build_completed") + #expect(json["success"] as? Bool == true) + #expect(json["timestamp"] != nil) + } + + @Test("Stage started event produces correct JSON output") + func testStageStartedEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let event = BuildEvent.stageStarted(stageName: "stage1", timestamp: Date()) + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let jsonLines = parseJSONLines(output) + + #expect(jsonLines.count == 1) + let json = jsonLines[0] + #expect(json["type"] as? String == "stage_started") + #expect(json["stage"] as? String == "stage1") + #expect(json["timestamp"] != nil) + } + + @Test("Stage completed event produces correct JSON output") + func testStageCompletedEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let event = BuildEvent.stageCompleted(stageName: "stage1", timestamp: Date()) + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let jsonLines = parseJSONLines(output) + + #expect(jsonLines.count == 1) + let json = jsonLines[0] + #expect(json["type"] as? String == "stage_completed") + #expect(json["stage"] as? String == "stage1") + #expect(json["timestamp"] != nil) + } + + @Test("Operation started event produces correct JSON output with operation number") + func testOperationStartedEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, stageId: "stage1", description: "Test operation") + let event = BuildEvent.operationStarted(context: context) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let jsonLines = parseJSONLines(output) + + #expect(jsonLines.count == 1) + let json = jsonLines[0] + #expect(json["type"] as? String == "operation_started") + #expect(json["operation"] as? String == "#1") + #expect(json["description"] as? String == "Test operation") + #expect(json["stage"] as? String == "stage1") + #expect(json["timestamp"] != nil) + } + + @Test("Operation finished event produces correct JSON output with duration") + func testOperationFinishedEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "Test operation") + + // First start the operation to assign a number + let startEvent = BuildEvent.operationStarted(context: context) + try await consumer.handle(startEvent) + + // Then finish it + let finishEvent = BuildEvent.operationFinished(context: context, duration: 1.5) + try await consumer.handle(finishEvent) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let jsonLines = parseJSONLines(output) + + #expect(jsonLines.count == 2) + let finishJson = jsonLines[1] + #expect(finishJson["type"] as? String == "operation_finished") + #expect(finishJson["operation"] as? String == "#1") + #expect(finishJson["duration"] as? Double == 1.5) + #expect(finishJson["timestamp"] != nil) + } + + @Test("Operation failed event produces correct JSON output with error details") + func testOperationFailedEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "Test operation") + let error = BuildEventError(type: .executionFailed, description: "Test error") + + // First start the operation to assign a number + let startEvent = BuildEvent.operationStarted(context: context) + try await consumer.handle(startEvent) + + // Then fail it + let failEvent = BuildEvent.operationFailed(context: context, error: error) + try await consumer.handle(failEvent) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let jsonLines = parseJSONLines(output) + + #expect(jsonLines.count == 2) + let failJson = jsonLines[1] + #expect(failJson["type"] as? String == "operation_failed") + #expect(failJson["operation"] as? String == "#1") + #expect(failJson["error"] as? String == "Test error") + #expect(failJson["error_type"] as? String == "executionFailed") + #expect(failJson["timestamp"] != nil) + } + + @Test("Operation cache hit event produces correct JSON output") + func testOperationCacheHitEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "Test operation") + let event = BuildEvent.operationCacheHit(context: context) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let jsonLines = parseJSONLines(output) + + #expect(jsonLines.count == 1) + let json = jsonLines[0] + #expect(json["type"] as? String == "operation_cache_hit") + #expect(json["operation"] as? String == "#1") + #expect(json["description"] as? String == "Test operation") + #expect(json["timestamp"] != nil) + } + + @Test("Operation progress event produces correct JSON output with progress fraction") + func testOperationProgressEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "Test operation") + + // First start the operation to assign a number + let startEvent = BuildEvent.operationStarted(context: context) + try await consumer.handle(startEvent) + + // Then progress it + let progressEvent = BuildEvent.operationProgress(context: context, fraction: 0.75) + try await consumer.handle(progressEvent) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let jsonLines = parseJSONLines(output) + + #expect(jsonLines.count == 2) + let progressJson = jsonLines[1] + #expect(progressJson["type"] as? String == "operation_progress") + #expect(progressJson["operation"] as? String == "#1") + #expect(progressJson["progress"] as? Double == 0.75) + #expect(progressJson["timestamp"] != nil) + } + + @Test("Operation log event produces correct JSON output with message") + func testOperationLogEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "Test operation") + + // First start the operation to assign a number + let startEvent = BuildEvent.operationStarted(context: context) + try await consumer.handle(startEvent) + + // Then log a message + let logEvent = BuildEvent.operationLog(context: context, message: "Test log message") + try await consumer.handle(logEvent) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let jsonLines = parseJSONLines(output) + + #expect(jsonLines.count == 2) + let logJson = jsonLines[1] + #expect(logJson["type"] as? String == "operation_log") + #expect(logJson["operation"] as? String == "#1") + #expect(logJson["message"] as? String == "Test log message") + #expect(logJson["timestamp"] != nil) + } + + @Test("IR event produces correct JSON output with source map information") + func testIREvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let nodeId = UUID() + let sourceMap = SourceMap(file: "test.swift", line: 42, column: 10, snippet: "let x = 1") + let context = ReportContext(nodeId: nodeId, stageId: "stage1", description: "IR test", sourceMap: sourceMap) + let event = BuildEvent.irEvent(context: context, type: .graphStarted) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let jsonLines = parseJSONLines(output) + + #expect(jsonLines.count == 1) + let json = jsonLines[0] + #expect(json["type"] as? String == "ir_event") + #expect(json["event_type"] as? String == "graph_started") + #expect(json["description"] as? String == "IR test") + #expect(json["node_id"] as? String == nodeId.uuidString) + #expect(json["stage_id"] as? String == "stage1") + #expect(json["timestamp"] != nil) + + // Check source map + let sourceMapJson = json["source_map"] as? [String: Any] + #expect(sourceMapJson != nil) + #expect(sourceMapJson?["file"] as? String == "test.swift") + #expect(sourceMapJson?["line"] as? Int == 42) + #expect(sourceMapJson?["column"] as? Int == 10) + #expect(sourceMapJson?["snippet"] as? String == "let x = 1") + } + + // MARK: - Operation Number Assignment Tests + + @Test("Operation number assignment assigns sequential numbers to operations") + func testOperationNumberAssignment() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let nodeId1 = UUID() + let nodeId2 = UUID() + let nodeId3 = UUID() + + let events = [ + BuildEvent.operationStarted(context: ReportContext(nodeId: nodeId1, description: "Op 1")), + BuildEvent.operationStarted(context: ReportContext(nodeId: nodeId2, description: "Op 2")), + BuildEvent.operationStarted(context: ReportContext(nodeId: nodeId3, description: "Op 3")), + BuildEvent.operationFinished(context: ReportContext(nodeId: nodeId1, description: "Op 1"), duration: 1.0), + BuildEvent.operationFinished(context: ReportContext(nodeId: nodeId2, description: "Op 2"), duration: 2.0), + BuildEvent.operationFinished(context: ReportContext(nodeId: nodeId3, description: "Op 3"), duration: 3.0), + ] + + for event in events { + try await consumer.handle(event) + } + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let jsonLines = parseJSONLines(output) + + #expect(jsonLines.count == 6) + + // Check operation numbers are assigned sequentially + #expect(jsonLines[0]["operation"] as? String == "#1") + #expect(jsonLines[1]["operation"] as? String == "#2") + #expect(jsonLines[2]["operation"] as? String == "#3") + #expect(jsonLines[3]["operation"] as? String == "#1") + #expect(jsonLines[4]["operation"] as? String == "#2") + #expect(jsonLines[5]["operation"] as? String == "#3") + } + + @Test("Operation number consistency maintains same number across operation lifecycle") + func testOperationNumberConsistency() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "Test operation") + + let events = [ + BuildEvent.operationStarted(context: context), + BuildEvent.operationProgress(context: context, fraction: 0.5), + BuildEvent.operationLog(context: context, message: "Progress update"), + BuildEvent.operationFinished(context: context, duration: 1.0), + ] + + for event in events { + try await consumer.handle(event) + } + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let jsonLines = parseJSONLines(output) + + #expect(jsonLines.count == 4) + + // All events should have the same operation number + for json in jsonLines { + #expect(json["operation"] as? String == "#1") + } + } + + // MARK: - Pretty Print Tests + + @Test("Pretty print formatting produces indented JSON with newlines") + func testPrettyPrintFormatting() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: true) + let consumer = JSONProgressConsumer(configuration: config) + + let event = BuildEvent.buildStarted(totalOperations: 5, stages: 2, timestamp: Date()) + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + + // Pretty printed JSON should contain newlines and indentation + #expect(output.contains("{\n")) + #expect(output.contains(" ")) + #expect(output.contains("\"stages\" : 2")) + #expect(output.contains("\"total_operations\" : 5")) + #expect(output.contains("\"type\" : \"build_started\"")) + } + + @Test("Compact formatting produces single-line JSON without indentation") + func testCompactFormatting() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let event = BuildEvent.buildStarted(totalOperations: 5, stages: 2, timestamp: Date()) + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + + // Compact JSON should be on a single line + let lines = output.split(separator: "\n") + #expect(lines.count == 1) + #expect(!output.contains(" ")) + } + + // MARK: - Edge Cases and Error Handling + + @Test("Operation event without node ID produces no output") + func testOperationEventWithoutNodeId() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let context = ReportContext(nodeId: nil, description: "Test operation") + let event = BuildEvent.operationStarted(context: context) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + + // Should produce no output when nodeId is nil + #expect(output.isEmpty) + } + + @Test("Operation finished without prior start produces no output") + func testOperationFinishedWithoutPriorStart() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "Test operation") + let event = BuildEvent.operationFinished(context: context, duration: 1.0) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + + // Should produce no output when operation wasn't started + #expect(output.isEmpty) + } + + @Test("IR event with minimal context produces basic JSON output") + func testIREventWithMinimalContext() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let context = ReportContext(nodeId: nil, description: "Minimal IR event") + let event = BuildEvent.irEvent(context: context, type: .graphStarted) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let jsonLines = parseJSONLines(output) + + #expect(jsonLines.count == 1) + let json = jsonLines[0] + #expect(json["type"] as? String == "ir_event") + #expect(json["event_type"] as? String == "graph_started") + #expect(json["description"] as? String == "Minimal IR event") + #expect(json["node_id"] == nil) + #expect(json["stage_id"] == nil) + #expect(json["source_map"] == nil) + } + + @Test("IR event with partial source map includes only available fields") + func testIREventWithPartialSourceMap() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let sourceMap = SourceMap(file: "test.swift", line: nil, column: nil, snippet: nil) + let context = ReportContext(nodeId: nil, description: "Partial source map", sourceMap: sourceMap) + let event = BuildEvent.irEvent(context: context, type: .graphStarted) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let jsonLines = parseJSONLines(output) + + #expect(jsonLines.count == 1) + let json = jsonLines[0] + let sourceMapJson = json["source_map"] as? [String: Any] + #expect(sourceMapJson != nil) + #expect(sourceMapJson?["file"] as? String == "test.swift") + #expect(sourceMapJson?["line"] == nil) + #expect(sourceMapJson?["column"] == nil) + #expect(sourceMapJson?["snippet"] == nil) + } + + // MARK: - Thread Safety Tests + + @Test("Thread safety with concurrent operations maintains unique operation numbers") + func testThreadSafetyWithConcurrentOperations() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = JSONProgressConsumer.Configuration(output: writeHandle, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let operationCount = 100 + let nodeIds = (0.. (FileHandle, FileHandle) { + let pipe = Pipe() + return (pipe.fileHandleForWriting, pipe.fileHandleForReading) + } + + private func readOutputAsString(from readHandle: FileHandle) -> String { + let data = readHandle.readDataToEndOfFile() + return String(data: data, encoding: .utf8) ?? "" + } + + private func parseLines(_ output: String) -> [String] { + output.split(separator: "\n").map(String.init).filter { !$0.isEmpty } + } + + // MARK: - Initialization Tests + + @Test("Default initialization uses standard output") + func defaultInitialization() { + let consumer = PlainProgressConsumer(configuration: PlainProgressConsumer.Configuration()) + + #expect(consumer.configuration.output == FileHandle.standardOutput) + } + + @Test("Initialization with custom configuration") + func initializationWithCustomConfiguration() { + let (writeHandle, _) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + #expect(consumer.configuration.output == writeHandle) + } + + // MARK: - Build Event Formatting Tests + + @Test("Build started event produces no output") + func buildStartedEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let event = BuildEvent.buildStarted(totalOperations: 5, stages: 2, timestamp: Date()) + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + + // BuildKit doesn't show explicit build start, so should be empty + #expect(output.isEmpty) + } + + @Test("Build completed event produces no output") + func buildCompletedEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let event = BuildEvent.buildCompleted(success: true, timestamp: Date()) + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + + // BuildKit doesn't show explicit build completion in plain output + #expect(output.isEmpty) + } + + @Test("Stage events produce no output") + func stageEvents() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let startEvent = BuildEvent.stageStarted(stageName: "stage1", timestamp: Date()) + let completeEvent = BuildEvent.stageCompleted(stageName: "stage1", timestamp: Date()) + + try await consumer.handle(startEvent) + try await consumer.handle(completeEvent) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + + // Stages are implicit in operation descriptions + #expect(output.isEmpty) + } + + @Test("Operation started event formats correctly") + func operationStartedEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, stageId: "stage1", description: "RUN apk add git") + let event = BuildEvent.operationStarted(context: context) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 1) + #expect(lines[0] == "#1 [stage1] RUN apk add git") + } + + @Test("Operation finished event shows duration") + func operationFinishedEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let startTime = Date() + let endTime = Date(timeInterval: 1.5, since: startTime) + let context = ReportContext(nodeId: nodeId, description: "RUN apk add git", timestamp: startTime) + let endContext = ReportContext(nodeId: nodeId, description: "RUN apk add git", timestamp: endTime) + + // Start operation first + let startEvent = BuildEvent.operationStarted(context: context) + try await consumer.handle(startEvent) + + // Then finish it + let finishEvent = BuildEvent.operationFinished(context: endContext, duration: 1.5) + try await consumer.handle(finishEvent) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 2) + #expect(lines[0] == "#1 [stage] RUN apk add git") + #expect(lines[1] == "#1 DONE 1.5s") + } + + @Test("Operation failed event shows error message") + func operationFailedEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "RUN apk add git") + let error = BuildEventError(type: .executionFailed, description: "Package not found") + + // Start operation first + let startEvent = BuildEvent.operationStarted(context: context) + try await consumer.handle(startEvent) + + // Then fail it + let failEvent = BuildEvent.operationFailed(context: context, error: error) + try await consumer.handle(failEvent) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 2) + #expect(lines[0] == "#1 [stage] RUN apk add git") + #expect(lines[1] == "#1 ERROR: Package not found") + } + + @Test("Operation cache hit event shows CACHED status") + func operationCacheHitEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, stageId: "stage1", description: "RUN apk add git") + let event = BuildEvent.operationCacheHit(context: context) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 2) + #expect(lines[0] == "#1 [stage1] RUN apk add git") + #expect(lines[1] == "#1 CACHED") + } + + @Test("Operation progress event shows percentage") + func operationProgressEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "RUN apk add git") + + // Start operation first + let startEvent = BuildEvent.operationStarted(context: context) + try await consumer.handle(startEvent) + + // Then show progress + let progressEvent = BuildEvent.operationProgress(context: context, fraction: 0.75) + try await consumer.handle(progressEvent) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 2) + #expect(lines[0] == "#1 [stage] RUN apk add git") + #expect(lines[1] == "#1 75% complete") + } + + @Test("Operation log event shows timestamped message") + func operationLogEvent() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let startTime = Date() + let logTime = Date(timeInterval: 0.245, since: startTime) + let context = ReportContext(nodeId: nodeId, description: "RUN apk add git", timestamp: startTime) + let logContext = ReportContext(nodeId: nodeId, description: "RUN apk add git", timestamp: logTime) + + // Start operation first + let startEvent = BuildEvent.operationStarted(context: context) + try await consumer.handle(startEvent) + + // Then log a message + let logEvent = BuildEvent.operationLog(context: logContext, message: "fetch https://dl-cdn.alpinelinux.org/alpine/...") + try await consumer.handle(logEvent) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 2) + #expect(lines[0] == "#1 [stage] RUN apk add git") + #expect(lines[1] == "#1 0.245 fetch https://dl-cdn.alpinelinux.org/alpine/...") + } + + // MARK: - Operation Description Formatting Tests + + @Test("FROM operation description formatting") + func fromOperationDescriptionFormatting() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "FROM alpine:latest") + let event = BuildEvent.operationStarted(context: context) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 1) + #expect(lines[0] == "#1 [internal] load metadata for alpine:latest") + } + + @Test("BaseImage operation description formatting") + func baseImageOperationDescriptionFormatting() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "BaseImage: ubuntu:20.04") + let event = BuildEvent.operationStarted(context: context) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 1) + #expect(lines[0] == "#1 [internal] load metadata for ubuntu:20.04") + } + + @Test("Stage name formatting") + func stageNameFormatting() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, stageId: "build-stage", description: "COPY . /app") + let event = BuildEvent.operationStarted(context: context) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 1) + #expect(lines[0] == "#1 [build-stage] COPY . /app") + } + + @Test("UUID stage name formatting") + func uuidStageNameFormatting() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, stageId: "stage-\(UUID().uuidString)", description: "RUN make") + let event = BuildEvent.operationStarted(context: context) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 1) + #expect(lines[0] == "#1 [stage] RUN make") + } + + @Test("Operation without stage ID") + func operationWithoutStageId() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "COPY . /app") + let event = BuildEvent.operationStarted(context: context) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 1) + #expect(lines[0] == "#1 [stage] COPY . /app") + } + + // MARK: - Duration Formatting Tests + + @Test("Duration formatting for sub-second durations") + func durationFormattingSubSecond() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let startTime = Date() + let endTime = Date(timeInterval: 0.123, since: startTime) + let context = ReportContext(nodeId: nodeId, description: "Test", timestamp: startTime) + let endContext = ReportContext(nodeId: nodeId, description: "Test", timestamp: endTime) + + let startEvent = BuildEvent.operationStarted(context: context) + try await consumer.handle(startEvent) + + let finishEvent = BuildEvent.operationFinished(context: endContext, duration: 0.123) + try await consumer.handle(finishEvent) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 2) + #expect(lines[1] == "#1 DONE 0.1s") + } + + @Test("Duration formatting for seconds") + func durationFormattingSeconds() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let startTime = Date() + let endTime = Date(timeInterval: 15.7, since: startTime) + let context = ReportContext(nodeId: nodeId, description: "Test", timestamp: startTime) + let endContext = ReportContext(nodeId: nodeId, description: "Test", timestamp: endTime) + + let startEvent = BuildEvent.operationStarted(context: context) + try await consumer.handle(startEvent) + + let finishEvent = BuildEvent.operationFinished(context: endContext, duration: 15.7) + try await consumer.handle(finishEvent) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 2) + #expect(lines[1] == "#1 DONE 15.7s") + } + + @Test("Duration formatting for minutes") + func durationFormattingMinutes() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let startTime = Date() + let endTime = Date(timeInterval: 125.0, since: startTime) // 2m5s + let context = ReportContext(nodeId: nodeId, description: "Test", timestamp: startTime) + let endContext = ReportContext(nodeId: nodeId, description: "Test", timestamp: endTime) + + let startEvent = BuildEvent.operationStarted(context: context) + try await consumer.handle(startEvent) + + let finishEvent = BuildEvent.operationFinished(context: endContext, duration: 125.0) + try await consumer.handle(finishEvent) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 2) + #expect(lines[1] == "#1 DONE 2m5s") + } + + // MARK: - IR Event Formatting Tests + + @Test("IR event analyzing") + func irEventAnalyzing() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let context = ReportContext(nodeId: UUID(), description: "Analyzing build dependencies") + let event = BuildEvent.irEvent(context: context, type: .analyzing) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 1) + #expect(lines[0] == "=> Analyzing build dependencies") + } + + @Test("IR event validating") + func irEventValidating() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let context = ReportContext(nodeId: UUID(), description: "Validating build graph") + let event = BuildEvent.irEvent(context: context, type: .validating) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 1) + #expect(lines[0] == "=> Validating build graph") + } + + @Test("IR event error") + func irEventError() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let context = ReportContext(nodeId: UUID(), description: "Invalid syntax") + let event = BuildEvent.irEvent(context: context, type: .error) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 1) + #expect(lines[0] == "ERROR: Invalid syntax") + } + + @Test("IR event error with source map") + func irEventErrorWithSourceMap() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let sourceMap = SourceMap(file: "Dockerfile", line: 15, column: 5, snippet: "RUN invalid-command") + let context = ReportContext(nodeId: UUID(), description: "Command not found", sourceMap: sourceMap) + let event = BuildEvent.irEvent(context: context, type: .error) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 1) + #expect(lines[0] == "ERROR: Command not found at Dockerfile:15") + } + + @Test("IR event warning") + func irEventWarning() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let context = ReportContext(nodeId: UUID(), description: "Deprecated instruction") + let event = BuildEvent.irEvent(context: context, type: .warning) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 1) + #expect(lines[0] == "WARNING: Deprecated instruction") + } + + @Test("IR event warning with source map") + func irEventWarningWithSourceMap() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let sourceMap = SourceMap(file: "Dockerfile", line: 8, column: nil, snippet: nil) + let context = ReportContext(nodeId: UUID(), description: "MAINTAINER is deprecated", sourceMap: sourceMap) + let event = BuildEvent.irEvent(context: context, type: .warning) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 1) + #expect(lines[0] == "WARNING: MAINTAINER is deprecated at Dockerfile:8") + } + + @Test("IR event graph events produce no output") + func irEventGraphEvents() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let events = [ + BuildEvent.irEvent(context: ReportContext(nodeId: UUID(), description: "Graph started"), type: .graphStarted), + BuildEvent.irEvent(context: ReportContext(nodeId: UUID(), description: "Graph completed"), type: .graphCompleted), + BuildEvent.irEvent(context: ReportContext(nodeId: UUID(), description: "Stage added"), type: .stageAdded), + BuildEvent.irEvent(context: ReportContext(nodeId: UUID(), description: "Node added"), type: .nodeAdded), + ] + + for event in events { + try await consumer.handle(event) + } + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + + // These events should not produce output in plain format + #expect(output.isEmpty) + } + + // MARK: - Operation Number Assignment Tests + + @Test("Operation number assignment") + func operationNumberAssignment() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId1 = UUID() + let nodeId2 = UUID() + let nodeId3 = UUID() + + let events = [ + BuildEvent.operationStarted(context: ReportContext(nodeId: nodeId1, description: "Op 1")), + BuildEvent.operationStarted(context: ReportContext(nodeId: nodeId2, description: "Op 2")), + BuildEvent.operationStarted(context: ReportContext(nodeId: nodeId3, description: "Op 3")), + ] + + for event in events { + try await consumer.handle(event) + } + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 3) + #expect(lines[0].hasPrefix("#1 ")) + #expect(lines[1].hasPrefix("#2 ")) + #expect(lines[2].hasPrefix("#3 ")) + } + + @Test("Operation number consistency") + func operationNumberConsistency() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let startTime = Date() + let progressTime = Date(timeInterval: 0.5, since: startTime) + let logTime = Date(timeInterval: 1.0, since: startTime) + let endTime = Date(timeInterval: 2.0, since: startTime) + + let events = [ + BuildEvent.operationStarted(context: ReportContext(nodeId: nodeId, description: "Test", timestamp: startTime)), + BuildEvent.operationProgress(context: ReportContext(nodeId: nodeId, description: "Test", timestamp: progressTime), fraction: 0.5), + BuildEvent.operationLog(context: ReportContext(nodeId: nodeId, description: "Test", timestamp: logTime), message: "Log message"), + BuildEvent.operationFinished(context: ReportContext(nodeId: nodeId, description: "Test", timestamp: endTime), duration: 2.0), + ] + + for event in events { + try await consumer.handle(event) + } + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 4) + #expect(lines[0].hasPrefix("#1 ")) + #expect(lines[1].hasPrefix("#1 ")) + #expect(lines[2].hasPrefix("#1 ")) + #expect(lines[3].hasPrefix("#1 ")) + } + + // MARK: - Edge Cases and Error Handling + + @Test("Operation event without node ID produces no output") + func operationEventWithoutNodeId() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let context = ReportContext(nodeId: nil, description: "Test operation") + let event = BuildEvent.operationStarted(context: context) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + + // Should produce no output when nodeId is nil + #expect(output.isEmpty) + } + + @Test("Operation finished without prior start produces no output") + func operationFinishedWithoutPriorStart() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "Test operation") + let event = BuildEvent.operationFinished(context: context, duration: 1.0) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + + // Should produce no output when operation wasn't started + #expect(output.isEmpty) + } + + @Test("Operation progress without prior start produces no output") + func operationProgressWithoutPriorStart() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "Test operation") + let event = BuildEvent.operationProgress(context: context, fraction: 0.5) + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + + // Should produce no output when operation wasn't started + #expect(output.isEmpty) + } + + @Test("Operation log without prior start produces no output") + func operationLogWithoutPriorStart() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "Test operation") + let event = BuildEvent.operationLog(context: context, message: "Log message") + + try await consumer.handle(event) + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + + // Should produce no output when operation wasn't started + #expect(output.isEmpty) + } + + @Test("Multiple operations with same node ID") + func multipleOperationsWithSameNodeId() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "Test operation") + + let events = [ + BuildEvent.operationStarted(context: context), + BuildEvent.operationStarted(context: context), // Same nodeId + BuildEvent.operationFinished(context: context, duration: 1.0), + ] + + for event in events { + try await consumer.handle(event) + } + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 3) + #expect(lines[0].hasPrefix("#1 ")) + #expect(lines[1].hasPrefix("#1 ")) // Same number reused + #expect(lines[2].hasPrefix("#1 ")) + } + + // MARK: - Thread Safety Tests + + @Test("Thread safety with concurrent operations") + func threadSafetyWithConcurrentOperations() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let operationCount = 100 + let nodeIds = (0.. Analyzing build graph") + #expect(lines[1] == "#1 [internal] load metadata for alpine:latest") + #expect(lines[2] == "#1 CACHED") + #expect(lines[3] == "#2 [stage1] RUN apk add --no-cache git") + #expect(lines[4] == "#2 0.245 fetch https://dl-cdn.alpinelinux.org/alpine/...") + #expect(lines[5] == "#2 DONE 1.2s") + #expect(lines[6] == "#3 [stage2] COPY . /app") + #expect(lines[7] == "#3 ERROR: No such file or directory") + + // Note: The failure line is now included in the count + } + + @Test("BuildKit-style output") + func buildKitStyleOutput() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId1 = UUID() + let nodeId2 = UUID() + + let events = [ + BuildEvent.operationStarted(context: ReportContext(nodeId: nodeId1, description: "BaseImage: alpine:latest")), + BuildEvent.operationCacheHit(context: ReportContext(nodeId: nodeId1, description: "BaseImage: alpine:latest")), + BuildEvent.operationStarted(context: ReportContext(nodeId: nodeId2, stageId: "base", description: "RUN apk add --no-cache git")), + BuildEvent.operationLog( + context: ReportContext(nodeId: nodeId2, description: "RUN apk add --no-cache git", timestamp: Date(timeInterval: 0.245, since: Date())), + message: "fetch https://dl-cdn.alpinelinux.org/alpine/..."), + BuildEvent.operationFinished( + context: ReportContext(nodeId: nodeId2, description: "RUN apk add --no-cache git", timestamp: Date(timeInterval: 1.2, since: Date())), duration: 1.2), + ] + + for event in events { + try await consumer.handle(event) + } + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 5) + + // Verify BuildKit-style output + #expect(lines[0] == "#1 [internal] load metadata for alpine:latest") + #expect(lines[1] == "#1 CACHED") + #expect(lines[2] == "#2 [base] RUN apk add --no-cache git") + #expect(lines[3] == "#2 0.245 fetch https://dl-cdn.alpinelinux.org/alpine/...") + #expect(lines[4] == "#2 DONE 1.2s") + } + + @Test("Consumer with reporter") + func consumerWithReporter() async throws { + let (writeHandle, readHandle) = createTestPipe() + let config = PlainProgressConsumer.Configuration(output: writeHandle) + let consumer = PlainProgressConsumer(configuration: config) + let reporter = Reporter(bufferSize: 100) + + // Start consuming in background + let consumeTask = Task { + do { + try await consumer.consume(reporter: reporter) + } catch { + Issue.record("Consumer failed: \(error)") + } + } + + // Report some events + let nodeId1 = UUID() + let nodeId2 = UUID() + let startTime = Date() + let finishTime = Date(timeInterval: 1.0, since: startTime) + let events = [ + BuildEvent.buildStarted(totalOperations: 2, stages: 1, timestamp: Date()), + BuildEvent.operationStarted(context: ReportContext(nodeId: nodeId1, description: "FROM alpine:latest")), + BuildEvent.operationCacheHit(context: ReportContext(nodeId: nodeId1, description: "FROM alpine:latest")), + BuildEvent.operationStarted(context: ReportContext(nodeId: nodeId2, description: "RUN apk add git", timestamp: startTime)), + BuildEvent.operationFinished(context: ReportContext(nodeId: nodeId2, description: "RUN apk add git", timestamp: finishTime), duration: 1.0), + BuildEvent.buildCompleted(success: true, timestamp: Date()), + ] + + for event in events { + await reporter.report(event) + } + + await reporter.finish() + await consumeTask.value + + writeHandle.closeFile() + let output = readOutputAsString(from: readHandle) + let lines = parseLines(output) + + #expect(lines.count == 4) + #expect(lines[0] == "#1 [internal] load metadata for alpine:latest") + #expect(lines[1] == "#1 CACHED") + #expect(lines[2] == "#2 [stage] RUN apk add git") + #expect(lines[3] == "#2 DONE 1.0s") + } + +} diff --git a/Tests/NativeBuilderTests/ContainerBuildReportingTests/ReportableErrorTests.swift b/Tests/NativeBuilderTests/ContainerBuildReportingTests/ReportableErrorTests.swift new file mode 100644 index 00000000..394f6e9f --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildReportingTests/ReportableErrorTests.swift @@ -0,0 +1,521 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation +import Testing + +@testable import ContainerBuildReporting + +@Suite("ReportableError Tests") +struct ReportableErrorTests { + + // MARK: - Test Error Types + + struct TestExecutionError: ReportableError { + let message: String + let category: ErrorCategory = .executionFailed + let errorDiagnostics: ErrorDiagnostics? + + var errorCategory: ErrorCategory { category } + var shortDescription: String { message } + var detailedDescription: String? { nil } + var diagnostics: ErrorDiagnostics { errorDiagnostics ?? ErrorDiagnostics() } + var underlyingError: Error? { nil } + + var description: String { message } + + func toBuildEventError() -> BuildEventError { + BuildEventError( + type: .executionFailed, + description: message, + diagnostics: diagnostics.toDictionary() + ) + } + } + + struct TestNetworkError: ReportableError { + let url: String + let statusCode: Int + + var errorCategory: ErrorCategory { .networkError } + var shortDescription: String { "Network request to \(url) failed with status code \(statusCode)" } + var detailedDescription: String? { "HTTP request failed" } + var underlyingError: Error? { nil } + + var diagnostics: ErrorDiagnostics { + ErrorDiagnostics( + operation: "HTTP Request", + path: url, + exitCode: statusCode + ) + } + + var description: String { + "Network request to \(url) failed with status code \(statusCode)" + } + + func toBuildEventError() -> BuildEventError { + BuildEventError( + type: .executionFailed, + description: description, + diagnostics: diagnostics.toDictionary() + ) + } + } + + // MARK: - Error Category Tests + + @Test("Error categories map to correct failure types") + func testErrorCategoryMapping() { + // Test the mapping logic based on the actual default implementation + let _ = TestExecutionError(message: "test", errorDiagnostics: nil) + + // Test execution failed category + let execFailedError = TestExecutionError(message: "test", errorDiagnostics: nil) + let buildError = execFailedError.toBuildEventError() + #expect(buildError.type == .executionFailed) + + // Test that the mapping works through the actual interface + let categories: [ErrorCategory] = [ + .executionFailed, + .commandNotFound, + .permissionDenied, + .timeout, + .cancelled, + .invalidConfiguration, + .missingDependency, + .incompatiblePlatform, + .resourceExhausted, + .diskFull, + .memoryExhausted, + .fileNotFound, + .fileAccessDenied, + .networkError, + .syntaxError, + .validationError, + .unsupportedFeature, + .cacheCorrupted, + .cacheMiss, + .unknown, + ] + + // Just verify that all categories are valid + for category in categories { + #expect(category.rawValue.isEmpty == false) + } + } + + @Test("Error categories have meaningful descriptions") + func testErrorCategoryDescriptions() { + let categories: [ErrorCategory] = [ + .executionFailed, + .commandNotFound, + .permissionDenied, + .timeout, + .cancelled, + .invalidConfiguration, + .missingDependency, + .incompatiblePlatform, + .resourceExhausted, + .diskFull, + .memoryExhausted, + .fileNotFound, + .fileAccessDenied, + .networkError, + .syntaxError, + .validationError, + .unsupportedFeature, + .cacheCorrupted, + .cacheMiss, + .unknown, + ] + + for category in categories { + let description = String(describing: category) + #expect(!description.isEmpty) + #expect(description != "ErrorCategory") + } + } + + // MARK: - ErrorDiagnostics Tests + + @Test("ErrorDiagnostics creation with all parameters") + func testErrorDiagnosticsFullCreation() { + let diagnostics = ErrorDiagnostics( + operation: "docker build", + path: "/path/to/Dockerfile", + exitCode: 127, + environment: ["PATH": "/usr/bin", "USER": "root"] + ) + + #expect(diagnostics.operation == "docker build") + #expect(diagnostics.path == "/path/to/Dockerfile") + #expect(diagnostics.exitCode == 127) + #expect(diagnostics.environment?["PATH"] == "/usr/bin") + #expect(diagnostics.environment?["USER"] == "root") + // Note: suggestion is no longer a separate property + } + + @Test("ErrorDiagnostics toDictionary conversion") + func testErrorDiagnosticsToDictionary() { + let diagnostics = ErrorDiagnostics( + operation: "container run", + path: "/var/lib/containers/image.tar", + exitCode: 1, + environment: ["CONTAINER_RUNTIME": "podman"] + ) + + let dict = diagnostics.toDictionary() + + #expect(dict["operation"] == "container run") + #expect(dict["path"] == "/var/lib/containers/image.tar") + #expect(dict["exitCode"] == "1") + #expect(dict["env.CONTAINER_RUNTIME"] == "podman") + } + + @Test("ErrorDiagnostics with nil values") + func testErrorDiagnosticsWithNilValues() { + let diagnostics = ErrorDiagnostics( + operation: nil, + path: nil, + exitCode: nil, + environment: nil + ) + + let dict = diagnostics.toDictionary() + #expect(dict.isEmpty) + } + + @Test("ErrorDiagnostics with partial values") + func testErrorDiagnosticsPartialValues() { + let diagnostics = ErrorDiagnostics( + operation: "build step", + path: nil, + exitCode: 0, + environment: nil + ) + + let dict = diagnostics.toDictionary() + #expect(dict.count == 2) + #expect(dict["operation"] == "build step") + #expect(dict["exitCode"] == "0") + #expect(dict["path"] == nil) + } + + @Test("ErrorDiagnostics with multiple environment variables") + func testErrorDiagnosticsMultipleEnvVars() { + let diagnostics = ErrorDiagnostics( + operation: "compile", + environment: [ + "CC": "clang", + "CFLAGS": "-O2", + "LDFLAGS": "-lm", + ] + ) + + let dict = diagnostics.toDictionary() + + #expect(dict["env.CC"] == "clang") + #expect(dict["env.CFLAGS"] == "-O2") + #expect(dict["env.LDFLAGS"] == "-lm") + } + + // MARK: - ReportableError Protocol Tests + + @Test("Custom error implements ReportableError") + func testCustomReportableError() { + let error = TestExecutionError( + message: "Command failed", + errorDiagnostics: ErrorDiagnostics( + operation: "docker build", + exitCode: 1 + ) + ) + + #expect(error.errorCategory == ErrorCategory.executionFailed) + #expect(error.description == "Command failed") + + let buildError = error.toBuildEventError() + #expect(buildError.description == "Command failed") + #expect(buildError.type == BuildEventError.FailureType.executionFailed) + #expect(buildError.diagnostics?["operation"] == "docker build") + #expect(buildError.diagnostics?["exitCode"] == "1") + } + + @Test("Network error with diagnostics") + func testNetworkErrorWithDiagnostics() { + let error = TestNetworkError( + url: "https://registry.example.com/v2/", + statusCode: 404 + ) + + #expect(error.errorCategory == .networkError) + // Note: suggestion is no longer a separate property + + let buildError = error.toBuildEventError() + #expect(buildError.type == .executionFailed) + #expect(buildError.diagnostics?["path"] == "https://registry.example.com/v2/") + #expect(buildError.diagnostics?["exitCode"] == "404") + } + + // MARK: - GenericReportableError Tests + + @Test("GenericReportableError wraps standard errors") + func testGenericReportableError() { + let nsError = NSError( + domain: "com.example.container", + code: 42, + userInfo: [NSLocalizedDescriptionKey: "Container not found"] + ) + + let reportableError = nsError.asReportableError() + + #expect(reportableError.errorCategory == .unknown) + #expect(reportableError.shortDescription.contains("Container not found")) + + let buildError = reportableError.toBuildEventError() + #expect(buildError.type == .executionFailed) + #expect(buildError.description.contains("Container not found")) + } + + @Test("Simple Swift error conversion") + func testSimpleErrorConversion() { + enum TestError: Error { + case simpleError + } + + let error = TestError.simpleError + let reportableError = error.asReportableError() + + #expect(reportableError.errorCategory == .unknown) + #expect(reportableError.shortDescription.contains("TestError")) + } + + @Test("LocalizedError conversion preserves message") + func testLocalizedErrorConversion() { + struct CustomError: LocalizedError { + var errorDescription: String? { + "Custom localized error message" + } + } + + let error = CustomError() + let reportableError = error.asReportableError() + + #expect(reportableError.shortDescription == "Custom localized error message") + } + + // MARK: - BuildEventError Conversion Tests + + @Test("BuildEventError preserves all diagnostics") + func testBuildEventErrorFullConversion() { + let diagnostics = ErrorDiagnostics( + operation: "layer extraction", + path: "/tmp/layer.tar.gz", + exitCode: 2, + environment: ["TMPDIR": "/tmp"] + ) + + let error = TestExecutionError( + message: "Failed to extract layer", + errorDiagnostics: diagnostics + ) + + let buildError = error.toBuildEventError() + + #expect(buildError.description == "Failed to extract layer") + #expect(buildError.type == .executionFailed) + #expect(buildError.diagnostics?.count == 4) + #expect(buildError.diagnostics?["operation"] == "layer extraction") + #expect(buildError.diagnostics?["path"] == "/tmp/layer.tar.gz") + #expect(buildError.diagnostics?["exitCode"] == "2") + #expect(buildError.diagnostics?["env.TMPDIR"] == "/tmp") + } + + @Test("BuildEventError with nil diagnostics") + func testBuildEventErrorNilDiagnostics() { + let error = TestExecutionError( + message: "Generic failure", + errorDiagnostics: nil + ) + + let buildError = error.toBuildEventError() + + #expect(buildError.description == "Generic failure") + #expect(buildError.type == .executionFailed) + #expect(buildError.diagnostics?.isEmpty == true) + } + + // MARK: - Edge Cases + + @Test("Empty diagnostics produces empty dictionary") + func testEmptyDiagnostics() { + let diagnostics = ErrorDiagnostics() + let dict = diagnostics.toDictionary() + + #expect(dict.isEmpty) + } + + @Test("Complex error hierarchy") + func testComplexErrorHierarchy() { + struct OuterError: Error { + let inner: Error + } + + struct InnerError: ReportableError { + var errorCategory: ErrorCategory { .validationError } + var diagnostics: ErrorDiagnostics { ErrorDiagnostics() } + + var shortDescription: String { "Inner validation error" } + + var description: String { shortDescription } + + func toBuildEventError() -> BuildEventError { + BuildEventError( + type: .invalidConfiguration, + description: description, + diagnostics: nil + ) + } + } + + let innerError = InnerError() + let outerError = OuterError(inner: innerError) + + let reportableError = outerError.asReportableError() + #expect(reportableError.errorCategory == .unknown) + #expect(reportableError.shortDescription.contains("OuterError")) + } + + // MARK: - Error Categorization Tests + + @Test("Proper category assignment for file errors") + func testFileCategoryAssignment() { + struct FileError: ReportableError { + let path: String + let isNotFound: Bool + + var errorCategory: ErrorCategory { + isNotFound ? .fileNotFound : .fileAccessDenied + } + + var diagnostics: ErrorDiagnostics { + ErrorDiagnostics(path: path) + } + + var shortDescription: String { + isNotFound ? "File not found: \(path)" : "Access denied: \(path)" + } + + var description: String { + shortDescription + } + + func toBuildEventError() -> BuildEventError { + BuildEventError( + type: .executionFailed, + description: description, + diagnostics: diagnostics.toDictionary() + ) + } + } + + let notFoundError = FileError(path: "/missing/file", isNotFound: true) + #expect(notFoundError.errorCategory == .fileNotFound) + #expect(notFoundError.toBuildEventError().type == .executionFailed) + + let accessError = FileError(path: "/protected/file", isNotFound: false) + #expect(accessError.errorCategory == .fileAccessDenied) + #expect(accessError.toBuildEventError().type == .executionFailed) + } + + @Test("Unknown category fallback") + func testUnknownCategoryFallback() { + struct UnknownError: ReportableError { + var errorCategory: ErrorCategory { .unknown } + var diagnostics: ErrorDiagnostics { ErrorDiagnostics() } + + var shortDescription: String { "Something went wrong" } + + var description: String { shortDescription } + + func toBuildEventError() -> BuildEventError { + BuildEventError( + type: .executionFailed, + description: description, + diagnostics: nil + ) + } + } + + let error = UnknownError() + let buildError = error.toBuildEventError() + + #expect(buildError.type == .executionFailed) + } + + @Test("Resource exhaustion errors") + func testResourceExhaustionErrors() { + struct ResourceError: ReportableError { + enum ResourceType { + case disk + case memory + case cpu + } + + let resourceType: ResourceType + let usage: String + + var errorCategory: ErrorCategory { + switch resourceType { + case .disk: return .diskFull + case .memory: return .memoryExhausted + case .cpu: return .resourceExhausted + } + } + + var diagnostics: ErrorDiagnostics { + ErrorDiagnostics( + operation: "resource check" + ) + } + + var shortDescription: String { + "\(resourceType) exhausted: \(usage)" + } + + var description: String { + shortDescription + } + + func toBuildEventError() -> BuildEventError { + BuildEventError( + type: .resourceExhausted, + description: description, + diagnostics: diagnostics.toDictionary() + ) + } + } + + let diskError = ResourceError(resourceType: .disk, usage: "99%") + #expect(diskError.errorCategory == .diskFull) + #expect(diskError.toBuildEventError().type == .resourceExhausted) + + let memoryError = ResourceError(resourceType: .memory, usage: "8GB/8GB") + #expect(memoryError.errorCategory == .memoryExhausted) + #expect(memoryError.toBuildEventError().type == .resourceExhausted) + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildReportingTests/SimpleNewTests.swift b/Tests/NativeBuilderTests/ContainerBuildReportingTests/SimpleNewTests.swift new file mode 100644 index 00000000..62f1ffb9 --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildReportingTests/SimpleNewTests.swift @@ -0,0 +1,79 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation +import Testing + +@testable import ContainerBuildReporting + +@Suite("Simple New Tests") +struct SimpleNewTests { + + @Test("BuildStatistics basic initialization") + func testBuildStatisticsBasic() { + let stats = BuildStatistics() + #expect(stats.startTime == nil) + #expect(stats.endTime == nil) + #expect(stats.duration == nil) + #expect(stats.totalOperations == 0) + } + + @Test("StageStatistics basic initialization") + func testStageStatisticsBasic() { + let stageStats = StageStatistics(name: "test") + #expect(stageStats.name == "test") + #expect(stageStats.startTime == nil) + #expect(stageStats.endTime == nil) + #expect(stageStats.duration == nil) + #expect(stageStats.operationCount == 0) + } + + @Test("Basic JSON output functionality") + func testBasicJSONOutput() async throws { + let pipe = Pipe() + let config = JSONProgressConsumer.Configuration(output: pipe.fileHandleForWriting, prettyPrint: false) + let consumer = JSONProgressConsumer(configuration: config) + + let event = BuildEvent.buildStarted(totalOperations: 1, stages: 1, timestamp: Date()) + try await consumer.handle(event) + + pipe.fileHandleForWriting.closeFile() + let data = pipe.fileHandleForReading.readDataToEndOfFile() + let output = String(data: data, encoding: .utf8) ?? "" + + #expect(output.contains("\"type\":\"build_started\"")) + #expect(output.contains("\"total_operations\":1")) + } + + @Test("Basic plain output functionality") + func testBasicPlainOutput() async throws { + let pipe = Pipe() + let config = PlainProgressConsumer.Configuration(output: pipe.fileHandleForWriting) + let consumer = PlainProgressConsumer(configuration: config) + + let nodeId = UUID() + let context = ReportContext(nodeId: nodeId, description: "Test operation") + let event = BuildEvent.operationStarted(context: context) + + try await consumer.handle(event) + + pipe.fileHandleForWriting.closeFile() + let data = pipe.fileHandleForReading.readDataToEndOfFile() + let output = String(data: data, encoding: .utf8) ?? "" + + #expect(output.contains("#1 [stage] Test operation")) + } +}