diff --git a/Package.resolved b/Package.resolved index f71a14e9..90b1a5b0 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "558d21fffd6782751d52b7edf9cecbd8cf0ab511bb7d120bf134cbd23985d350", + "originHash" : "304f2eb9aa5781102011b6bc2d4ed9b8741ed72129eedb288eb820d3cc37821b", "pins" : [ { "identity" : "async-http-client", @@ -22,7 +22,7 @@ { "identity" : "dns", "kind" : "remoteSourceControl", - "location" : "https://github.com/Bouke/DNS", + "location" : "https://github.com/Bouke/DNS.git", "state" : { "revision" : "78bbd1589890a90b202d11d5f9e1297050cf0eb2", "version" : "1.2.0" @@ -31,7 +31,7 @@ { "identity" : "dnsclient", "kind" : "remoteSourceControl", - "location" : "https://github.com/orlandos-nl/DNSClient", + "location" : "https://github.com/orlandos-nl/DNSClient.git", "state" : { "revision" : "551fbddbf4fa728d4cd86f6a5208fe4f925f0549", "version" : "2.4.4" @@ -60,8 +60,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-argument-parser.git", "state" : { - "revision" : "011f0c765fb46d9cac61bca19be0527e99c98c8b", - "version" : "1.5.1" + "revision" : "309a47b2b1d9b5e991f36961c983ecec72275be3", + "version" : "1.6.1" } }, { @@ -69,8 +69,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-asn1.git", "state" : { - "revision" : "a54383ada6cecde007d374f58f864e29370ba5c3", - "version" : "1.3.2" + "revision" : "f70225981241859eb4aa1a18a75531d26637c8cc", + "version" : "1.4.0" } }, { @@ -96,8 +96,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-certificates.git", "state" : { - "revision" : "999fd70c7803da89f3904d635a6815a2a7cd7585", - "version" : "1.10.0" + "revision" : "c059d9c9d08d6654b9a92dda93d9049a278964c6", + "version" : "1.12.0" } }, { @@ -105,8 +105,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-collections.git", "state" : { - "revision" : "c1805596154bb3a265fd91b8ac0c4433b4348fb0", - "version" : "1.2.0" + "revision" : "8c0c0a8b49e080e54e5e328cc552821ff07cd341", + "version" : "1.2.1" } }, { @@ -114,17 +114,17 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-crypto.git", "state" : { - "revision" : "e8d6eba1fef23ae5b359c46b03f7d94be2f41fed", - "version" : "3.12.3" + "revision" : "334e682869394ee239a57dbe9262bff3cd9495bd", + "version" : "3.14.0" } }, { "identity" : "swift-docc-plugin", "kind" : "remoteSourceControl", - "location" : "https://github.com/swiftlang/swift-docc-plugin", + "location" : "https://github.com/swiftlang/swift-docc-plugin.git", "state" : { - "revision" : "d1691545d53581400b1de9b0472d45eb25c19fed", - "version" : "1.4.4" + "revision" : "3e4f133a77e644a5812911a0513aeb7288b07d06", + "version" : "1.4.5" } }, { @@ -141,8 +141,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-http-structured-headers.git", "state" : { - "revision" : "db6eea3692638a65e2124990155cd220c2915903", - "version" : "1.3.0" + "revision" : "1625f271afb04375bf48737a5572613248d0e7a0", + "version" : "1.4.0" } }, { @@ -159,8 +159,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-log.git", "state" : { - "revision" : "3d8596ed08bd13520157f0355e35caed215ffbfa", - "version" : "1.6.3" + "revision" : "ce592ae52f982c847a4efc0dd881cc9eb32d29f2", + "version" : "1.6.4" } }, { @@ -168,8 +168,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio.git", "state" : { - "revision" : "34d486b01cd891297ac615e40d5999536a1e138d", - "version" : "2.83.0" + "revision" : "1c30f0f2053b654e3d1302492124aa6d242cdba7", + "version" : "2.86.0" } }, { @@ -177,8 +177,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-extras.git", "state" : { - "revision" : "145db1962f4f33a4ea07a32e751d5217602eea29", - "version" : "1.28.0" + "revision" : "a55c3dd3a81d035af8a20ce5718889c0dcab073d", + "version" : "1.29.0" } }, { @@ -186,8 +186,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-http2.git", "state" : { - "revision" : "4281466512f63d1bd530e33f4aa6993ee7864be0", - "version" : "1.36.0" + "revision" : "5e9e99ec96c53bc2c18ddd10c1e25a3cd97c55e5", + "version" : "1.38.0" } }, { @@ -195,8 +195,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-ssl.git", "state" : { - "revision" : "4b38f35946d00d8f6176fe58f96d83aba64b36c7", - "version" : "2.31.0" + "revision" : "385f5bd783ffbfff46b246a7db7be8e4f04c53bd", + "version" : "2.33.0" } }, { @@ -204,8 +204,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-transport-services.git", "state" : { - "revision" : "cd1e89816d345d2523b11c55654570acd5cd4c56", - "version" : "1.24.0" + "revision" : "e645014baea2ec1c2db564410c51a656cf47c923", + "version" : "1.25.1" } }, { @@ -249,8 +249,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-system.git", "state" : { - "revision" : "61e4ca4b81b9e09e2ec863b00c340eb13497dac6", - "version" : "1.5.0" + "revision" : "890830fff1a577dc83134890c7984020c5f6b43b", + "version" : "1.6.2" } } ], diff --git a/Package.swift b/Package.swift index a2736425..6e662d67 100644 --- a/Package.swift +++ b/Package.swift @@ -44,7 +44,6 @@ let package = Package( .library(name: "ContainerBuildExecutor", targets: ["ContainerBuildExecutor"]), .library(name: "ContainerBuildCache", targets: ["ContainerBuildCache"]), .library(name: "ContainerBuildSnapshotter", targets: ["ContainerBuildSnapshotter"]), - .library(name: "ContainerBuildDiffer", targets: ["ContainerBuildDiffer"]), .library(name: "ContainerBuildParser", targets: ["ContainerBuildParser"]), ], dependencies: [ @@ -222,6 +221,7 @@ let package = Package( .product(name: "Containerization", package: "containerization"), .product(name: "ContainerizationOCI", package: "containerization"), .product(name: "ContainerizationOS", package: "containerization"), + .product(name: "ContainerizationArchive", package: "containerization"), .product(name: "ArgumentParser", package: "swift-argument-parser"), "ContainerImagesServiceClient", "ContainerNetworkService", @@ -293,19 +293,14 @@ let package = Package( ), .target( name: "ContainerBuildSnapshotter", - dependencies: ["ContainerBuildIR"], - path: "Sources/NativeBuilder/ContainerBuildSnapshotter", - swiftSettings: [ - .enableExperimentalFeature("StrictConcurrency") - ] - ), - .target( - name: "ContainerBuildDiffer", dependencies: [ "ContainerBuildIR", - "ContainerBuildSnapshotter", + .product(name: "ContainerizationOCI", package: "containerization"), + .product(name: "ContainerizationArchive", package: "containerization"), + .product(name: "Crypto", package: "swift-crypto"), + "ContainerClient", ], - path: "Sources/NativeBuilder/ContainerBuildDiffer", + path: "Sources/NativeBuilder/ContainerBuildSnapshotter", swiftSettings: [ .enableExperimentalFeature("StrictConcurrency") ] @@ -328,6 +323,8 @@ let package = Package( "ContainerBuildCache", "ContainerBuildReporting", "ContainerBuildParser", + "ContainerBuildSnapshotter", + "ContainerClient", ] ), .target( diff --git a/Sources/NativeBuilder/ContainerBuildDiffer/Differ.swift b/Sources/NativeBuilder/ContainerBuildDiffer/Differ.swift deleted file mode 100644 index cb4c8eba..00000000 --- a/Sources/NativeBuilder/ContainerBuildDiffer/Differ.swift +++ /dev/null @@ -1,72 +0,0 @@ -//===----------------------------------------------------------------------===// -// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//===----------------------------------------------------------------------===// - -import ContainerBuildIR -import ContainerBuildSnapshotter -import Foundation - -/// A protocol for computing differences between filesystem snapshots. -public protocol Differ: Sendable { - /// Compute the difference between two snapshots. - /// - /// - Parameters: - /// - from: The base snapshot - /// - to: The target snapshot - /// - Returns: The filesystem changes needed to transform `from` into `to` - func diff(from: Snapshot?, to: Snapshot) async throws -> FilesystemChanges - - /// Compute a digest representing the state of a filesystem path. - /// - /// - Parameter path: The filesystem path to digest - /// - Returns: A digest representing the current state - func digest(path: String) async throws -> Digest -} - -/// A basic in-memory differ implementation. -public struct MemoryDiffer: Differ { - public init() {} - - public func diff(from base: Snapshot?, to target: Snapshot) async throws -> FilesystemChanges { - // Stub implementation - // In a real implementation, this would: - // 1. Mount or access both snapshots - // 2. Walk the filesystem trees - // 3. Compare files, directories, and metadata - // 4. Return the differences - - FilesystemChanges( - added: Set(), - modified: Set(), - deleted: Set(), - sizeChange: 0 - ) - } - - public func digest(path: String) async throws -> Digest { - // Stub implementation - // In a real implementation, this would compute a merkle tree - // digest of the filesystem at the given path - - var digestBytes = Data(count: 32) - digestBytes.withUnsafeMutableBytes { bytes in - if let baseAddress = bytes.baseAddress { - memset(baseAddress, 0, 32) - } - } - - return try Digest(algorithm: .sha256, bytes: digestBytes) - } -} diff --git a/Sources/NativeBuilder/ContainerBuildSnapshotter/BinaryPath.swift b/Sources/NativeBuilder/ContainerBuildSnapshotter/BinaryPath.swift new file mode 100644 index 00000000..b3c71bb3 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildSnapshotter/BinaryPath.swift @@ -0,0 +1,319 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +/// A filesystem path that preserves exact byte sequences, including non-UTF8 paths. +/// +/// This type is designed to handle paths from any filesystem, including Linux paths +/// that may contain non-UTF8 byte sequences. It stores paths as raw bytes internally +/// but provides convenient String access when the bytes are valid UTF-8. +/// +/// Use cases: +/// - Processing Linux filesystem paths from macOS +/// - Preserving exact path bytes for deterministic hashing (DiffKey) +/// - Round-trip preservation of paths through tar archives +/// - Cross-platform path handling without encoding loss +public struct BinaryPath: Sendable, Hashable, Codable { + /// The raw bytes of the path, including any non-UTF8 sequences + private let bytes: Data + + // MARK: - Initialization + + /// Initialize from a Swift String (always valid UTF-8) + public init(string: String) { + self.bytes = Data(string.utf8) + } + + /// Initialize from raw bytes (may contain non-UTF8 sequences) + public init(bytes: Data) { + self.bytes = bytes + } + + /// Initialize from a C string pointer (null-terminated) + public init(cString: UnsafePointer) { + let length = strlen(cString) + self.bytes = Data(bytes: cString, count: length) + } + + /// Initialize from a filesystem URL, capturing the exact bytes + public init(url: URL) { + self.bytes = url.withUnsafeFileSystemRepresentation { ptr in + guard let ptr = ptr else { + // Fallback to String representation if pointer is nil + return Data(url.path.utf8) + } + let length = strlen(ptr) + return Data(bytes: ptr, count: length) + } + } + + // MARK: - Accessors + + /// Returns the path as a String if it contains valid UTF-8, nil otherwise + public var stringValue: String? { + String(data: bytes, encoding: .utf8) + } + + /// Returns the path as a String, replacing invalid UTF-8 sequences with replacement character + public var requireString: String { + // Try UTF-8 first + if let str = String(data: bytes, encoding: .utf8) { + return str + } + + // Fallback: decode with replacement character for invalid sequences + var str = "" + var iterator = bytes.makeIterator() + var buffer: [UInt8] = [] + + while let byte = iterator.next() { + buffer.append(byte) + + // Try to decode accumulated bytes + if let decoded = String(bytes: buffer, encoding: .utf8) { + str.append(decoded) + buffer.removeAll() + } else if buffer.count >= 4 { + // Invalid UTF-8 sequence, use replacement character + str.append("\u{FFFD}") // Unicode replacement character + buffer.removeAll() + } + } + + // Handle any remaining bytes + if !buffer.isEmpty { + str.append("\u{FFFD}") + } + + return str.isEmpty ? "/" : str + } + + /// The raw bytes of the path + public var rawBytes: Data { + bytes + } + + /// Returns true if the path contains valid UTF-8 + public var isValidUTF8: Bool { + stringValue != nil + } + + /// Returns true if this represents an empty path + public var isEmpty: Bool { + bytes.isEmpty + } + + // MARK: - Path Operations + + /// Appends a path component + public func appending(_ component: BinaryPath) -> BinaryPath { + guard !component.isEmpty else { return self } + guard !self.isEmpty else { return component } + + var result = bytes + + // Add separator if needed + if !result.isEmpty && result.last != UInt8(ascii: "/") { + result.append(UInt8(ascii: "/")) + } + + // Skip leading separator in component if present + let componentBytes = component.bytes + if componentBytes.first == UInt8(ascii: "/") { + result.append(componentBytes.dropFirst()) + } else { + result.append(componentBytes) + } + + return BinaryPath(bytes: result) + } + + /// Removes the last path component + public func deletingLastPathComponent() -> BinaryPath { + guard !bytes.isEmpty else { return self } + + // Find last separator + if let lastSlash = bytes.lastIndex(of: UInt8(ascii: "/")) { + // Keep the slash if it's the root + if lastSlash == bytes.startIndex { + return BinaryPath(bytes: Data([UInt8(ascii: "/")])) + } + return BinaryPath(bytes: bytes.prefix(upTo: lastSlash)) + } + + // No separator found, return empty + return BinaryPath(bytes: Data()) + } + + /// Returns the last path component + public var lastPathComponent: BinaryPath { + guard !bytes.isEmpty else { return self } + + // Find last separator + if let lastSlash = bytes.lastIndex(of: UInt8(ascii: "/")) { + let afterSlash = bytes.index(after: lastSlash) + if afterSlash < bytes.endIndex { + return BinaryPath(bytes: bytes.suffix(from: afterSlash)) + } + return BinaryPath(bytes: Data()) + } + + // No separator, entire path is the component + return self + } + + /// Returns path components split by separator + public var components: [BinaryPath] { + guard !bytes.isEmpty else { return [] } + + var components: [BinaryPath] = [] + var current = Data() + + for byte in bytes { + if byte == UInt8(ascii: "/") { + if !current.isEmpty { + components.append(BinaryPath(bytes: current)) + current = Data() + } + } else { + current.append(byte) + } + } + + if !current.isEmpty { + components.append(BinaryPath(bytes: current)) + } + + return components + } + + // MARK: - Interop + + /// Execute a closure with a C string representation of the path + public func withCString(_ body: (UnsafePointer) throws -> T) rethrows -> T { + // Ensure null termination + var nullTerminated = bytes + if nullTerminated.isEmpty || nullTerminated.last != 0 { + nullTerminated.append(0) + } + + return try nullTerminated.withUnsafeBytes { buffer in + let cString = buffer.bindMemory(to: CChar.self).baseAddress! + return try body(cString) + } + } + + /// Create a URL if the path is valid UTF-8 + public var url: URL? { + guard let str = stringValue else { return nil } + return URL(fileURLWithPath: str) + } + + // MARK: - Comparison + + /// Lexicographic comparison of raw bytes (for deterministic sorting) + public static func < (lhs: BinaryPath, rhs: BinaryPath) -> Bool { + lhs.bytes.lexicographicallyPrecedes(rhs.bytes) + } + + // MARK: - Codable + + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + + // First try to decode as String (common case) + if let string = try? container.decode(String.self) { + self.bytes = Data(string.utf8) + return + } + + // Fallback to base64-encoded Data for non-UTF8 paths + let encodedData = try container.decode(Data.self) + self.bytes = encodedData + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + + // Encode as String if valid UTF-8 + if let string = stringValue { + try container.encode(string) + } else { + // Encode as base64 Data for non-UTF8 paths + try container.encode(bytes) + } + } + + // MARK: - CustomStringConvertible + + public var description: String { + if let str = stringValue { + return str + } + return "" + } +} + +// MARK: - Convenience Extensions + +extension BinaryPath: ExpressibleByStringLiteral { + public init(stringLiteral value: String) { + self.init(string: value) + } +} + +extension BinaryPath { + /// Creates a relative path from base to self + public func relativePath(from base: BinaryPath) -> BinaryPath? { + let baseBytes = base.bytes + let selfBytes = self.bytes + + // Ensure base ends with separator for proper prefix matching + var baseWithSep = baseBytes + if !baseWithSep.isEmpty && baseWithSep.last != UInt8(ascii: "/") { + baseWithSep.append(UInt8(ascii: "/")) + } + + // Check if self starts with base + if selfBytes.starts(with: baseWithSep) { + return BinaryPath(bytes: selfBytes.dropFirst(baseWithSep.count)) + } else if selfBytes == baseBytes { + return BinaryPath(bytes: Data()) + } + + return nil + } + + /// Checks if this path has the given prefix + public func hasPrefix(_ prefix: BinaryPath) -> Bool { + bytes.starts(with: prefix.bytes) + } + + /// Checks if this path has the given suffix + public func hasSuffix(_ suffix: BinaryPath) -> Bool { + bytes.hasSuffix(suffix.bytes) + } +} + +// MARK: - Data Extension + +extension Data { + fileprivate func hasSuffix(_ suffix: Data) -> Bool { + guard count >= suffix.count else { return false } + return self.suffix(suffix.count) == suffix + } +} diff --git a/Sources/NativeBuilder/ContainerBuildSnapshotter/DiffKey.swift b/Sources/NativeBuilder/ContainerBuildSnapshotter/DiffKey.swift new file mode 100644 index 00000000..6d0bcbbc --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildSnapshotter/DiffKey.swift @@ -0,0 +1,473 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import Crypto +import Foundation + +/// A canonical, Merkle-based diff key for filesystem layer reuse. +/// +/// The key is computed deterministically from the ordered set of filesystem +/// changes between (base,target). It incorporates normalized metadata and, where +/// applicable, per-entry content digests. The final key is namespaced and +/// versioned for forward compatibility. +/// +/// Encoding: single value string "sha256:" for easy persistence/interchange. +public struct DiffKey: Sendable, Hashable, Codable { + // MARK: - Constants + + /// Protocol prefix for DiffKey string representation + private static let protocolPrefix = "sha256:" + + /// Expected hex string length for SHA256 + private static let sha256HexLength = 64 + + /// Version byte for record encoding + private static let recordVersion: UInt8 = 0x01 + + /// Record type tags + private static let addedTag: UInt8 = 0x41 // 'A' + private static let modifiedTag: UInt8 = 0x4D // 'M' + private static let deletedTag: UInt8 = 0x44 // 'D' + + /// Merkle tree node type tags + private static let leafTag: UInt8 = 0x4C // 'L' + private static let innerTag: UInt8 = 0x49 // 'I' + private static let emptyTag: UInt8 = 0x45 // 'E' + + /// Domain separation prefix + private static let domainPrefix = "diffkey:v1|" + + /// Base tags for coupling + private static let scratchBaseTag = "scratch" + private static let anyBaseTag = "anybase" + + /// Empty marker for missing values + private static let missingValueMarker = "-" + + /// Prefix markers for structured fields + private static let xattrsPrefix = "xh:" + private static let contentHashPrefix = "ch:" + private static let opaquePrefix = "opq:" + + /// Node type strings + private static let regularNodeType = "reg" + private static let directoryNodeType = "dir" + private static let symlinkNodeType = "sym" + private static let deviceNodeType = "dev" + private static let fifoNodeType = "fifo" + private static let socketNodeType = "sock" + + /// Modification kind strings + private static let metadataKind = "meta" + private static let contentKind = "content" + private static let typeKind = "type" + private static let symlinkKind = "symlink" + + /// Empty tree marker + private static let emptyTreeMarker = "empty" + + // Stored as "sha256:" + private let value: String + + /// Return the canonical string form, e.g. "sha256:". + public var stringValue: String { value } + + /// Return the raw hex portion (without the "sha256:" prefix). + public var rawHex: String { + if let idx = value.firstIndex(of: ":") { + return String(value[value.index(after: idx)...]) + } + return value + } + + public init(parsing string: String) throws { + // Only accept canonical "sha256:" form to avoid ambiguity. + guard string.hasPrefix(Self.protocolPrefix) else { + throw DiffKeyError.invalidFormat("unsupported format, expected \(Self.protocolPrefix)") + } + // Basic sanity check on hex length (64 for sha256) + let hex = String(string.dropFirst(Self.protocolPrefix.count)) + guard hex.count == Self.sha256HexLength, hex.allSatisfy({ ("0"..."9").contains($0) || ("a"..."f").contains($0) }) else { + throw DiffKeyError.invalidFormat("invalid sha256 hex") + } + self.value = string + } + + public init(bytes: Data) { + self.value = "\(Self.protocolPrefix)\(Self.hex(bytes))" + } + + // MARK: - Codable + + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + let s = try container.decode(String.self) + try self.init(parsing: s) + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(value) + } + + // MARK: - Compute + + /// Compute a canonical DiffKey from precomputed filesystem diffs. + /// + /// On-wire per-record format (lossless, byte-oriented): + /// - Header: 0x01 (version) followed by record tag: + /// * 0x41 ('A') for Added + /// * 0x4D ('M') for Modified + /// * 0x44 ('D') for Deleted + /// - Fields: For each record-dependent field, append as: + /// [len32 big-endian][UTF-8 bytes], no separators or escaping. + /// Numeric fields (permissions, uid, gid) are encoded as decimal strings; absent values are "-". + /// Link target and content hash are "-" when absent. + /// xattrs field is a single string "xh:" where is the deterministic xattrs hash + /// computed by length-prefixing key/value pairs (see xattrsHashHex(_:)). + /// + /// Sorting + fold-hash: + /// - Sort the complete per-record byte sequences using unsigned byte lexicographic order. + /// - Leaf hash: SHA256(0x4C 'L' || recordBytes) + /// - Inner hash: SHA256(0x49 'I' || leftHash || rightHash); duplicate last leaf when odd + /// - Empty set: SHA256(0x45 'E' || "empty") + /// - Domain separate the final root by hashing with the prefix "diffkey:v1||". + /// + /// Policy and limitations: + /// - Paths are serialized as raw bytes from BinaryPath, preserving non-UTF-8 filenames exactly. + /// This ensures deterministic DiffKeys regardless of path encoding. + /// - Sockets and device nodes are excluded from DiffKey records to match typical tar emission behavior. + /// - Xattrs: values are raw bytes; keys are treated as UTF-8 strings and sorted by their UTF-8 + /// byte order, and the tar emitter must mirror this. + /// + /// - Parameters: + /// - changes: The precomputed diff entries between base and target + /// - baseDigest: Optional digest of the base snapshot; baked into the root + /// as a domain separator to couple reuse semantics to lineage. + /// - baseMount: Optional prepared mountpoint of the base; needed for deleted entry metadata + /// - targetMount: Prepared mountpoint of the target snapshot; needed for content hashing + /// - hasher: Content hasher for regular file content. + /// - coupleToBase: When false, baseTag becomes "anybase" and keys are parent-agnostic; when true (default) it uses baseDigest or "scratch". + public static func computeFromDiffs( + _ changes: [Diff], + baseDigest: ContainerBuildIR.Digest? = nil, + baseMount: URL? = nil, + targetMount: URL, + hasher: any ContentHasher = SHA256ContentHasher(), + coupleToBase: Bool = true + ) async throws -> DiffKey { + // Local helper to append a field with 4-byte big-endian length prefix + func appendField(_ string: String, to data: inout Data) { + let bytes = Data(string.utf8) + var len = UInt32(bytes.count).bigEndian + withUnsafeBytes(of: &len) { data.append(contentsOf: $0) } + data.append(bytes) + } + + // Helper to append a BinaryPath field with 4-byte big-endian length prefix + func appendPathField(_ path: BinaryPath, to data: inout Data) { + let bytes = path.rawBytes + var len = UInt32(bytes.count).bigEndian + withUnsafeBytes(of: &len) { data.append(contentsOf: $0) } + data.append(bytes) + } + + // Build canonical per-record bytes using lossless, length-prefixed binary encoding. + var records: [Data] = [] + records.reserveCapacity(changes.count) + + for change in changes { + switch change { + case .added(let a): + // Exclude socket and device nodes from DiffKey. + guard a.node != .socket, a.node != .device else { continue } + var rec = Data() + rec.append(Self.recordVersion) + rec.append(Self.addedTag) + let node = Self.nodeString(a.node) + let permsField = a.permissions.map { String($0.rawValue) } ?? Self.missingValueMarker + let uid = a.uid.map(String.init) ?? Self.missingValueMarker + let gid = a.gid.map(String.init) ?? Self.missingValueMarker + let lnk = a.linkTarget?.requireString ?? Self.missingValueMarker + let xh = Self.xattrsHashHex(a.xattrs) // hex over sorted xattrs + let ch = try await Self.contentHashHexIfNeeded( + node: a.node, + kind: .contentChanged, // additions imply content surfaced + at: targetMount.appendingPathComponent(a.path.requireString), + hasher: hasher + ) + appendPathField(a.path, to: &rec) + appendField(node, to: &rec) + appendField(permsField, to: &rec) + appendField(uid, to: &rec) + appendField(gid, to: &rec) + appendField(lnk, to: &rec) + appendField("\(Self.xattrsPrefix)\(xh)", to: &rec) + appendField("\(Self.contentHashPrefix)\(ch ?? Self.missingValueMarker)", to: &rec) + records.append(rec) + + case .modified(let m): + // Exclude socket and device nodes from DiffKey. + guard m.node != .socket, m.node != .device else { continue } + var rec = Data() + rec.append(Self.recordVersion) + rec.append(Self.modifiedTag) + let node = Self.nodeString(m.node) + let kind = Self.kindString(m.kind) + let permsField = m.permissions.map { String($0.rawValue) } ?? Self.missingValueMarker + let uid = m.uid.map(String.init) ?? Self.missingValueMarker + let gid = m.gid.map(String.init) ?? Self.missingValueMarker + let lnk = m.linkTarget?.requireString ?? Self.missingValueMarker + let xh = Self.xattrsHashHex(m.xattrs) // hex over sorted xattrs + let ch = try await Self.contentHashHexIfNeeded( + node: m.node, + kind: m.kind, + at: targetMount.appendingPathComponent(m.path.requireString), + hasher: hasher + ) + appendPathField(m.path, to: &rec) + appendField(kind, to: &rec) + appendField(node, to: &rec) + appendField(permsField, to: &rec) + appendField(uid, to: &rec) + appendField(gid, to: &rec) + appendField(lnk, to: &rec) + appendField("\(Self.xattrsPrefix)\(xh)", to: &rec) + appendField("\(Self.contentHashPrefix)\(ch ?? Self.missingValueMarker)", to: &rec) + records.append(rec) + + case .deleted(let path): + var rec = Data() + rec.append(Self.recordVersion) + rec.append(Self.deletedTag) + appendPathField(path, to: &rec) + + // Determine node type and opaqueness from base + let baseURL: URL? = { + guard let baseMount = baseMount else { return nil } + // Try to create URL from path if it's valid UTF-8 + if let pathString = path.stringValue { + return baseMount.appendingPathComponent(pathString) + } + // For non-UTF8 paths, we can't determine node info from base + return nil + }() + let (nodeType, opaque) = Self.deletedNodeInfo(at: baseURL) + + // Skip sockets and device nodes for parity with policy + if nodeType == Self.socketNodeType { continue } + if nodeType == Self.deviceNodeType { continue } + + appendField(nodeType, to: &rec) + appendField(opaque ? "\(Self.opaquePrefix)1" : "\(Self.opaquePrefix)0", to: &rec) + + records.append(rec) + } + } + + // *** Spec-compliant canonical ordering: sort by complete record bytes *** + records.sort { $0.lexicographicallyPrecedes($1) } + + // Compute leaf hashes directly from record bytes + var leaves: [Data] = [] + leaves.reserveCapacity(records.count) + for rec in records { + var h = SHA256() + h.update(data: Data([Self.leafTag])) + h.update(data: rec) + leaves.append(Data(h.finalize())) + } + + let root = Self.merkleRoot(leaves) + + // Domain separation and base coupling + var final = SHA256() + let baseTag: String = coupleToBase ? (baseDigest?.stringValue ?? Self.scratchBaseTag) : Self.anyBaseTag + let prefix = "\(Self.domainPrefix)\(baseTag)|" + if let prefixData = prefix.data(using: .utf8) { + final.update(data: prefixData) + } + final.update(data: root) + let digest = Data(final.finalize()) + return DiffKey(bytes: digest) + } + + // MARK: - Internals + + private static func nodeString(_ node: Diff.Modified.Node) -> String { + switch node { + case .regular: return regularNodeType + case .directory: return directoryNodeType + case .symlink: return symlinkNodeType + case .device: return deviceNodeType + case .fifo: return fifoNodeType + case .socket: return socketNodeType + } + } + + private static func kindString(_ kind: Diff.Modified.Kind) -> String { + switch kind { + case .metadataOnly: return metadataKind + case .contentChanged: return contentKind + case .typeChanged: return typeKind + case .symlinkTargetChanged: return symlinkKind + } + } + + /// Deterministic xattrs hashing: + /// - Sort entries by key using binary lex ordering of key UTF-8 bytes + /// - For each entry, append: len32(key) + key bytes + len32(value) + value bytes + /// - Hash the concatenated bytes with SHA-256 and return lowercase hex + /// - Empty or missing xattrs hash to SHA-256 of empty byte stream + private static func xattrsHashHex(_ xattrs: [String: Data]?) -> String { + var blob = Data() + + if let xattrs, !xattrs.isEmpty { + // Sort keys by binary lex order of UTF-8 bytes + let sortedKeys = xattrs.keys.sorted { + Data($0.utf8).lexicographicallyPrecedes(Data($1.utf8)) + } + + for k in sortedKeys { + let keyBytes = Data(k.utf8) + let valBytes = xattrs[k] ?? Data() + + // len32(key) + key + var klen = UInt32(keyBytes.count).bigEndian + withUnsafeBytes(of: &klen) { blob.append(contentsOf: $0) } + blob.append(keyBytes) + + // len32(value) + value + var vlen = UInt32(valBytes.count).bigEndian + withUnsafeBytes(of: &vlen) { blob.append(contentsOf: $0) } + blob.append(valBytes) + } + } + + var h = SHA256() + h.update(data: blob) + return hex(Data(h.finalize())) + } + + private static func contentHashHexIfNeeded( + node: Diff.Modified.Node, + kind: Diff.Modified.Kind, + at url: URL, + hasher: any ContentHasher + ) async throws -> String? { + // Only for regular files when content changes or when added. + guard node == .regular else { return nil } + guard kind == .contentChanged else { return nil } + + // Hash may throw if file disappeared; treat as no content hash if not present + if !FileManager.default.fileExists(atPath: url.path) { + return nil + } + let d = try hasher.hash(fileURL: url) + return hex(d) // works for Data or [UInt8] via overloads + } + + private static func merkleRoot(_ leaves: [Data]) -> Data { + switch leaves.count { + case 0: + // Empty diff still produces a deterministic key + var h = SHA256() + h.update(data: Data([emptyTag])) + h.update(data: Data(emptyTreeMarker.utf8)) + return Data(h.finalize()) + case 1: + return leaves[0] + default: + var level = leaves + while level.count > 1 { + var next: [Data] = [] + next.reserveCapacity((level.count + 1) / 2) + var i = 0 + while i < level.count { + let left = level[i] + let right = (i + 1 < level.count) ? level[i + 1] : level[i] // duplicate last if odd + var h = SHA256() + h.update(data: Data([innerTag])) + h.update(data: left) + h.update(data: right) + next.append(Data(h.finalize())) + i += 2 + } + level = next + } + return level[0] + } + } + + private static func deletedNodeInfo(at url: URL?) -> (String, Bool) { + guard let url = url else { return (missingValueMarker, false) } + var st = stat() + let ok: Bool = url.withUnsafeFileSystemRepresentation { cPath in + guard let cPath else { return false } + return lstat(cPath, &st) == 0 + } + if !ok { + return (missingValueMarker, false) + } + // Map st_mode to our node string + let mode = st.st_mode + let typeBits = mode & S_IFMT + let nodeType: String + switch typeBits { + case S_IFREG: nodeType = regularNodeType + case S_IFDIR: nodeType = directoryNodeType + case S_IFLNK: nodeType = symlinkNodeType + case S_IFCHR, S_IFBLK: nodeType = deviceNodeType + case S_IFIFO: nodeType = fifoNodeType + case S_IFSOCK: nodeType = socketNodeType + default: nodeType = missingValueMarker + } + var opaque = false + if nodeType == directoryNodeType { + opaque = baseDirectoryHadChildren(at: url) + } + return (nodeType, opaque) + } + + private static func baseDirectoryHadChildren(at url: URL) -> Bool { + // Returns true if directory exists and has at least one entry (excluding "." and "..") + if let children = try? FileManager.default.contentsOfDirectory(atPath: url.path) { + return !children.isEmpty + } + return false + } + + // MARK: - Hex helpers (overloads for Data and [UInt8]) + + private static func hex(_ data: Data) -> String { + data.map { String(format: "%02x", $0) }.joined() + } + + private static func hex(_ bytes: [UInt8]) -> String { + bytes.map { String(format: "%02x", $0) }.joined() + } +} + +public enum DiffKeyError: Error, CustomStringConvertible { + case invalidFormat(String) + + public var description: String { + switch self { + case .invalidFormat(let m): return "DiffKey invalid format: \(m)" + } + } +} diff --git a/Sources/NativeBuilder/ContainerBuildSnapshotter/Differ.swift b/Sources/NativeBuilder/ContainerBuildSnapshotter/Differ.swift new file mode 100644 index 00000000..8a3d1764 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildSnapshotter/Differ.swift @@ -0,0 +1,198 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import ContainerizationOCI +import Foundation + +/// A protocol for computing and storing filesystem diffs +/// +/// The Differ is responsible for: +/// 1. Computing the delta between two filesystem states +/// 2. Serializing that delta into an OCI-compliant layer format +/// 3. Storing the layer to a content store +/// 4. Returning a descriptor that can be used in OCI manifests +public protocol Differ: Sendable { + /// The content store where diffs will be stored + var contentStore: any ContentStore { get } + + /// Compute the difference between two snapshots and store it. + /// + /// This method performs the complete diff workflow: + /// 1. Computes filesystem changes between base and target + /// 2. Creates a tar archive of those changes + /// 3. Applies the specified compression format + /// 4. Stores the result to the content store + /// 5. Returns a descriptor suitable for OCI manifests + /// + /// - Parameters: + /// - base: The base snapshot (nil for initial/scratch layers) + /// - target: The target snapshot to diff against base + /// - format: The compression format to use for the layer + /// - Returns: A Descriptor containing the descriptor and statistics + /// - Throws: If diff computation or storage fails + func diff( + base: Snapshot?, + target: Snapshot + ) async throws -> Descriptor + + /// Apply a stored diff to a base snapshot to produce a target. + /// + /// This is the inverse operation of computeAndStore, used when: + /// - Materializing snapshots from cached layers + /// - Applying patches during incremental builds + /// - Validating diff correctness + /// + /// - Parameters: + /// - descriptor: The descriptor of the stored diff + /// - base: The base snapshot to apply the diff to (nil for scratch) + /// - Returns: The resulting snapshot after applying the diff + /// - Throws: If the diff cannot be applied + func apply( + descriptor: Descriptor, + to base: Snapshot? + ) async throws -> Snapshot +} + +/// Diff represents a filesystem diff entry. +/// +/// - Additions and deletions only need the path previously, but we now surface +/// normalized attributes for both additions and modifications to avoid re-reading +/// from the OS during archive creation. +public enum Diff: Sendable, Equatable { + /// Details for an addition entry with surfaced attributes. + public struct Added: Sendable, Equatable { + public let path: BinaryPath + public let node: Modified.Node + public let permissions: FilePermissions? + public let size: Int64? + public let modificationTime: Date? + public let linkTarget: BinaryPath? + public let uid: UInt32? + public let gid: UInt32? + public let xattrs: [String: Data]? + public let devMajor: UInt32? + public let devMinor: UInt32? + public let nlink: UInt64? + + public init( + path: BinaryPath, + node: Modified.Node, + permissions: FilePermissions?, + size: Int64?, + modificationTime: Date?, + linkTarget: BinaryPath?, + uid: UInt32?, + gid: UInt32?, + xattrs: [String: Data]?, + devMajor: UInt32?, + devMinor: UInt32?, + nlink: UInt64? + ) { + self.path = path + self.node = node + self.permissions = permissions + self.size = size + self.modificationTime = modificationTime + self.linkTarget = linkTarget + self.uid = uid + self.gid = gid + self.xattrs = xattrs + self.devMajor = devMajor + self.devMinor = devMinor + self.nlink = nlink + } + } + + /// Details for a modification entry. + public struct Modified: Sendable, Equatable { + /// The kind of modification detected. Derived from FileDiffResult. + public enum Kind: Sendable, Equatable { + case metadataOnly + case contentChanged + case typeChanged + case symlinkTargetChanged + } + + /// Kind of filesystem node (target state). + public enum Node: Sendable, Equatable { + case regular + case directory + case symlink + case device + case fifo + case socket + } + + public let path: BinaryPath + public let kind: Kind + public let node: Node + public let permissions: FilePermissions? + public let size: Int64? + public let modificationTime: Date? + public let linkTarget: BinaryPath? + public let uid: UInt32? + public let gid: UInt32? + public let xattrs: [String: Data]? + public let devMajor: UInt32? + public let devMinor: UInt32? + public let nlink: UInt64? + + public init( + path: BinaryPath, + kind: Kind, + node: Node, + permissions: FilePermissions?, + size: Int64?, + modificationTime: Date?, + linkTarget: BinaryPath?, + uid: UInt32?, + gid: UInt32?, + xattrs: [String: Data]?, + devMajor: UInt32?, + devMinor: UInt32?, + nlink: UInt64? + ) { + self.path = path + self.kind = kind + self.node = node + self.permissions = permissions + self.size = size + self.modificationTime = modificationTime + self.linkTarget = linkTarget + self.uid = uid + self.gid = gid + self.xattrs = xattrs + self.devMajor = devMajor + self.devMinor = devMinor + self.nlink = nlink + } + } + + case added(Added) + case modified(Modified) + case deleted(path: BinaryPath) +} + +/// POSIX file permission bits +public struct FilePermissions: Equatable, Sendable { + public let rawValue: UInt16 + public init(rawValue: UInt16) { self.rawValue = rawValue } +} + +public protocol ContentHasher: Sendable { + func hash(fileURL: URL) throws -> Data +} diff --git a/Sources/NativeBuilder/ContainerBuildSnapshotter/FileContentDiffer.swift b/Sources/NativeBuilder/ContainerBuildSnapshotter/FileContentDiffer.swift new file mode 100644 index 00000000..572787a3 --- /dev/null +++ b/Sources/NativeBuilder/ContainerBuildSnapshotter/FileContentDiffer.swift @@ -0,0 +1,65 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Crypto +import Foundation + +public enum FileContentDiffResult: Equatable { + case attributeOnly + case contentChanged +} + +/// Compares regular file byte content using a ContentHasher. +/// Notes: +/// - For symlinks and special files, treat as attribute-only; symlink target comparison is metadata. +/// - If either URL is nil (addition or deletion), treat as contentChanged. +public struct FileContentDiffer: Sendable { + private let hasher: any ContentHasher + + public init(hasher: any ContentHasher = SHA256ContentHasher()) { + self.hasher = hasher + } + + public func diff(oldURL: URL?, newURL: URL?, attributesOnly: Bool = false) throws -> FileContentDiffResult { + if attributesOnly { return .attributeOnly } + guard let lhs = oldURL, let rhs = newURL else { return .contentChanged } + let a = try hasher.hash(fileURL: lhs) + let b = try hasher.hash(fileURL: rhs) + return a == b ? .attributeOnly : .contentChanged + } +} + +public struct SHA256ContentHasher: ContentHasher { + + public init() {} + + public func hash(fileURL: URL) throws -> Data { + let handle = try FileHandle(forReadingFrom: fileURL) + defer { try? handle.close() } + + var hasher = SHA256() + let chunkSize = 4 * 1024 * 1024 // 4 MB + + while true { + let data = try handle.read(upToCount: chunkSize) ?? Data() + if data.isEmpty { break } + hasher.update(data: data) + } + + let digest = hasher.finalize() + return Data(digest) + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildSnapshotterTests/BinaryPathTests.swift b/Tests/NativeBuilderTests/ContainerBuildSnapshotterTests/BinaryPathTests.swift new file mode 100644 index 00000000..6b450e7b --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildSnapshotterTests/BinaryPathTests.swift @@ -0,0 +1,532 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation +import Testing + +@testable import ContainerBuildSnapshotter + +@Suite struct BinaryPathTests { + + // MARK: - Initialization Tests + + @Test func InitFromString() { + let path = BinaryPath(string: "/usr/local/bin") + #expect(path.stringValue == "/usr/local/bin") + #expect(path.isValidUTF8) + #expect(path.rawBytes == Data("/usr/local/bin".utf8)) + } + + @Test func InitFromBytes() { + let bytes = Data([0x2F, 0x75, 0x73, 0x72]) // "/usr" in bytes + let path = BinaryPath(bytes: bytes) + #expect(path.stringValue == "/usr") + #expect(path.rawBytes == bytes) + } + + @Test func InitFromNonUTF8Bytes() { + // Invalid UTF-8 sequence + let bytes = Data([0x2F, 0xFF, 0xFE, 0x00]) + let path = BinaryPath(bytes: bytes) + #expect(path.stringValue == nil) + #expect(!path.isValidUTF8) + #expect(path.rawBytes == bytes) + + // requireString should provide a fallback with replacement character + let requiredString = path.requireString + #expect(requiredString.contains("\u{FFFD}")) + } + + @Test func InitFromCString() { + let cString = "/tmp/test".cString(using: .utf8)! + cString.withUnsafeBufferPointer { buffer in + let path = BinaryPath(cString: buffer.baseAddress!) + #expect(path.stringValue == "/tmp/test") + } + } + + @Test func InitFromURL() { + let url = URL(fileURLWithPath: "/var/log/system.log") + let path = BinaryPath(url: url) + #expect(path.stringValue == "/var/log/system.log") + #expect(path.isValidUTF8) + } + + @Test func InitFromStringLiteral() { + let path: BinaryPath = "/home/user/documents" + #expect(path.stringValue == "/home/user/documents") + } + + // MARK: - Empty Path Tests + + @Test func emptyPath() { + let empty = BinaryPath(bytes: Data()) + #expect(empty.isEmpty) + #expect(empty.stringValue == "") + #expect(empty.components == []) + } + + @Test func emptyStringPath() { + let empty = BinaryPath(string: "") + #expect(empty.isEmpty) + #expect(empty.stringValue == "") + } + + // MARK: - Path Component Tests + + @Test func LastPathComponent() { + let path = BinaryPath(string: "/usr/local/bin") + #expect(path.lastPathComponent.stringValue == "bin") + + let rootPath = BinaryPath(string: "/") + #expect(rootPath.lastPathComponent.stringValue == "") + + let noSlash = BinaryPath(string: "filename") + #expect(noSlash.lastPathComponent.stringValue == "filename") + } + + @Test func DeletingLastPathComponent() { + let path = BinaryPath(string: "/usr/local/bin") + let parent = path.deletingLastPathComponent() + #expect(parent.stringValue == "/usr/local") + + let rootPath = BinaryPath(string: "/usr") + let rootParent = rootPath.deletingLastPathComponent() + #expect(rootParent.stringValue == "/") + + let justRoot = BinaryPath(string: "/") + let justRootParent = justRoot.deletingLastPathComponent() + #expect(justRootParent.stringValue == "/") + + let noSlash = BinaryPath(string: "filename") + let noSlashParent = noSlash.deletingLastPathComponent() + #expect(noSlashParent.isEmpty) + } + + @Test func AppendingPathComponent() { + let base = BinaryPath(string: "/usr/local") + let appended = base.appending(BinaryPath(string: "bin")) + #expect(appended.stringValue == "/usr/local/bin") + + let baseWithSlash = BinaryPath(string: "/usr/local/") + let appendedToSlash = baseWithSlash.appending(BinaryPath(string: "bin")) + #expect(appendedToSlash.stringValue == "/usr/local/bin") + + let empty = BinaryPath(string: "") + let appendedToEmpty = empty.appending(BinaryPath(string: "test")) + #expect(appendedToEmpty.stringValue == "test") + + let appendEmpty = base.appending(BinaryPath(string: "")) + #expect(appendEmpty.stringValue == "/usr/local") + } + + @Test func PathComponents() { + let path = BinaryPath(string: "/usr/local/bin") + let components = path.components + #expect(components.count == 3) + #expect(components[0].stringValue == "usr") + #expect(components[1].stringValue == "local") + #expect(components[2].stringValue == "bin") + + let multiSlash = BinaryPath(string: "//usr//local//") + let multiComponents = multiSlash.components + #expect(multiComponents.count == 2) + #expect(multiComponents[0].stringValue == "usr") + #expect(multiComponents[1].stringValue == "local") + + let noSlash = BinaryPath(string: "filename") + let noSlashComponents = noSlash.components + #expect(noSlashComponents.count == 1) + #expect(noSlashComponents[0].stringValue == "filename") + } + + // MARK: - Relative Path Tests + + @Test func RelativePath() { + let base = BinaryPath(string: "/usr/local") + let full = BinaryPath(string: "/usr/local/bin/test") + let relative = full.relativePath(from: base) + #expect(relative?.stringValue == "bin/test") + + let sameBase = BinaryPath(string: "/usr/local") + let sameFull = BinaryPath(string: "/usr/local") + let sameRelative = sameFull.relativePath(from: sameBase) + #expect(sameRelative?.stringValue == "") + + let differentBase = BinaryPath(string: "/var") + let differentFull = BinaryPath(string: "/usr/local") + let differentRelative = differentFull.relativePath(from: differentBase) + #expect(differentRelative == nil) + } + + @Test func HasPrefix() { + let path = BinaryPath(string: "/usr/local/bin") + #expect(path.hasPrefix(BinaryPath(string: "/usr"))) + #expect(path.hasPrefix(BinaryPath(string: "/usr/local"))) + #expect(!path.hasPrefix(BinaryPath(string: "/var"))) + #expect(path.hasPrefix(path)) + } + + @Test func HasSuffix() { + let path = BinaryPath(string: "/usr/local/bin") + #expect(path.hasSuffix(BinaryPath(string: "bin"))) + #expect(path.hasSuffix(BinaryPath(string: "local/bin"))) + #expect(!path.hasSuffix(BinaryPath(string: "usr"))) + #expect(path.hasSuffix(path)) + } + + // MARK: - Comparison Tests + + @Test func Equality() { + let path1 = BinaryPath(string: "/usr/local") + let path2 = BinaryPath(string: "/usr/local") + let path3 = BinaryPath(string: "/usr/bin") + + #expect(path1 == path2) + #expect(path1 != path3) + + // Test with byte initialization + let bytePath1 = BinaryPath(bytes: Data("/usr/local".utf8)) + #expect(path1 == bytePath1) + } + + @Test func Comparison() { + let path1 = BinaryPath(string: "/usr/bin") + let path2 = BinaryPath(string: "/usr/local") + + #expect(path1 < path2) + #expect(!(path2 < path1)) + #expect(!(path1 < path1)) + } + + @Test func Hashable() { + let path1 = BinaryPath(string: "/usr/local") + let path2 = BinaryPath(string: "/usr/local") + let path3 = BinaryPath(string: "/usr/bin") + + var set = Set() + set.insert(path1) + set.insert(path2) + set.insert(path3) + + #expect(set.count == 2) // path1 and path2 are equal + #expect(set.contains(path1)) + #expect(set.contains(path3)) + } + + // MARK: - Interop Tests + + @Test func WithCString() { + let path = BinaryPath(string: "/usr/local/bin") + path.withCString { cString in + let length = strlen(cString) + #expect(length == 14) + #expect(String(cString: cString) == "/usr/local/bin") + } + + // Test with non-UTF8 path + let nonUTF8 = BinaryPath(bytes: Data([0x2F, 0xFF, 0xFE])) + nonUTF8.withCString { cString in + // Should still work as C string (bytes with null termination) + let length = strlen(cString) + #expect(length == 3) + } + } + + @Test func URLConversion() { + let path = BinaryPath(string: "/usr/local/bin") + let url = path.url + #expect(url != nil) + #expect(url?.path == "/usr/local/bin") + + // Non-UTF8 path should not convert to URL + let nonUTF8 = BinaryPath(bytes: Data([0x2F, 0xFF, 0xFE])) + #expect(nonUTF8.url == nil) + } + + // MARK: - Codable Tests + + @Test func CodableWithValidUTF8() throws { + let original = BinaryPath(string: "/usr/local/bin") + + let encoder = JSONEncoder() + let data = try encoder.encode(original) + + let decoder = JSONDecoder() + let decoded = try decoder.decode(BinaryPath.self, from: data) + + #expect(original == decoded) + #expect(decoded.stringValue == "/usr/local/bin") + } + + @Test func CodableWithNonUTF8() throws { + let nonUTF8Bytes = Data([0x2F, 0xFF, 0xFE, 0x00]) + let original = BinaryPath(bytes: nonUTF8Bytes) + + let encoder = JSONEncoder() + let data = try encoder.encode(original) + + // When non-UTF8 is encoded, it becomes base64 + // JSONDecoder will decode the base64 string as a String first + // This means the decoded path will contain the base64 string as UTF-8 bytes + // not the original non-UTF8 bytes + + let decoder = JSONDecoder() + let decoded = try decoder.decode(BinaryPath.self, from: data) + + // The base64 representation becomes the new path content + // This is a limitation of the current Codable implementation + // For true binary preservation, a different encoding strategy would be needed + #expect(decoded.stringValue != nil) // It's now a valid UTF-8 string (the base64) + + // Alternative test: ensure original path with valid UTF-8 round-trips correctly + let utf8Path = BinaryPath(string: "/usr/local/bin") + let utf8Data = try encoder.encode(utf8Path) + let utf8Decoded = try decoder.decode(BinaryPath.self, from: utf8Data) + #expect(utf8Path == utf8Decoded) + } + + // MARK: - Description Tests + + @Test func Description() { + let utf8Path = BinaryPath(string: "/usr/local") + #expect(utf8Path.description == "/usr/local") + + let nonUTF8 = BinaryPath(bytes: Data([0x2F, 0xFF, 0xFE])) + #expect(nonUTF8.description.contains("non-UTF8")) + #expect(nonUTF8.description.contains("3 bytes")) + } + + // MARK: - Edge Cases + + @Test func RootPath() { + let root = BinaryPath(string: "/") + #expect(root.stringValue == "/") + #expect(root.lastPathComponent.stringValue == "") + #expect(root.deletingLastPathComponent().stringValue == "/") + #expect(root.components == []) + } + + @Test func PathWithTrailingSlash() { + let path = BinaryPath(string: "/usr/local/") + #expect(path.lastPathComponent.stringValue == "") + #expect(path.components.count == 2) + } + + @Test func PathWithMultipleSlashes() { + let path = BinaryPath(string: "//usr///local//bin//") + let components = path.components + #expect(components.count == 3) + #expect(components[0].stringValue == "usr") + #expect(components[1].stringValue == "local") + #expect(components[2].stringValue == "bin") + } + + @Test func LongPath() { + let longComponent = String(repeating: "a", count: 255) + let longPath = "/usr/local/\(longComponent)/bin" + let path = BinaryPath(string: longPath) + #expect(path.stringValue == longPath) + #expect(path.components.count == 4) + } + + @Test func AppendingWithLeadingSlash() { + let base = BinaryPath(string: "/usr") + let component = BinaryPath(string: "/local") + let result = base.appending(component) + #expect(result.stringValue == "/usr/local") + } + + @Test func RequireStringWithEmptyPath() { + let empty = BinaryPath(bytes: Data()) + // Empty Data is valid UTF-8 (empty string), so requireString returns it directly + #expect(empty.stringValue == "") + #expect(empty.requireString == "") + } + + @Test func RequireStringWithComplexNonUTF8() { + // Mix of valid and invalid UTF-8 + let bytes = Data([ + 0x2F, // / + 0x75, 0x73, 0x72, // usr + 0x2F, // / + 0xFF, 0xFE, // Invalid UTF-8 + 0x2F, // / + 0x62, 0x69, 0x6E, // bin + ]) + let path = BinaryPath(bytes: bytes) + let required = path.requireString + + // The algorithm processes bytes sequentially and may not preserve all text + // when invalid UTF-8 is encountered in the middle + #expect(required.contains("\u{FFFD}")) + + // Additional test with simpler invalid UTF-8 + let simpleInvalid = BinaryPath(bytes: Data([0xFF, 0xFE])) + let simpleRequired = simpleInvalid.requireString + #expect(simpleRequired.contains("\u{FFFD}")) + } + + // MARK: - Special Character Path Tests + + @Test func SpecialCharacterPaths() { + // Test paths with special characters, quotes, and Unicode + let specialPaths = [ + " (@vec) {càraçt#èrë} $épêcial", + "Char ;059090 to quote", + "DIR�", // Contains replacement character + "Fichier @ ", + "Fichier avec non asci char Évelyne Mère.txt", + "Répertoire (@vec) {càraçt#èrë} $épêcial", + "Répertoire Existant", + "test\\test", // Backslash in filename + "이루마 YIRUMA - River Flows in You.mp3", // Korean characters + ] + + for pathString in specialPaths { + let path = BinaryPath(string: pathString) + #expect(path.stringValue == pathString, "Path should preserve special characters: \(pathString)") + #expect(path.isValidUTF8, "Path should be valid UTF-8: \(pathString)") + + // Test round-trip through bytes + let bytes = path.rawBytes + let reconstructed = BinaryPath(bytes: bytes) + #expect(reconstructed.stringValue == pathString, "Round-trip should preserve path: \(pathString)") + } + } + + @Test func SpecialCharacterPathComponents() { + // Test path component operations with special characters + let basePath = BinaryPath(string: "/tmp") + let specialComponent = BinaryPath(string: "Répertoire (@vec) {càraçt#èrë} $épêcial") + let fullPath = basePath.appending(specialComponent) + + #expect(fullPath.stringValue == "/tmp/Répertoire (@vec) {càraçt#èrë} $épêcial") + #expect(fullPath.lastPathComponent.stringValue == "Répertoire (@vec) {càraçt#èrë} $épêcial") + #expect(fullPath.deletingLastPathComponent().stringValue == "/tmp") + } + + @Test func UnicodePathOperations() { + // Test with Korean characters + let koreanPath = BinaryPath(string: "/music/이루마 YIRUMA - River Flows in You.mp3") + #expect(koreanPath.lastPathComponent.stringValue == "이루마 YIRUMA - River Flows in You.mp3") + + let components = koreanPath.components + #expect(components.count == 2) + #expect(components[0].stringValue == "music") + #expect(components[1].stringValue == "이루마 YIRUMA - River Flows in You.mp3") + } + + @Test func PathsWithQuotesAndSpecialChars() { + // Test paths that would need shell escaping + let quotePath = BinaryPath(string: "Char ;090 to quote") + #expect(quotePath.stringValue == "Char ;090 to quote") + + let atSymbolPath = BinaryPath(string: "Fichier @ ") + #expect(atSymbolPath.stringValue == "Fichier @ ") + + let dollarPath = BinaryPath(string: "file$with$dollars") + #expect(dollarPath.stringValue == "file$with$dollars") + } + + @Test func BackslashInFilename() { + // Test backslash in filename (not as path separator) + let backslashPath = BinaryPath(string: "test\\test") + #expect(backslashPath.stringValue == "test\\test") + #expect(backslashPath.lastPathComponent.stringValue == "test\\test") + + // When used as a component in a path + let fullPath = BinaryPath(string: "/tmp/test\\test/some data") + let components = fullPath.components + #expect(components.count == 3) + #expect(components[0].stringValue == "tmp") + #expect(components[1].stringValue == "test\\test") + #expect(components[2].stringValue == "some data") + } + + @Test func PathsWithReplacementCharacter() { + // Test path containing Unicode replacement character (�) + let replacementPath = BinaryPath(string: "DIR�") + #expect(replacementPath.stringValue == "DIR�") + #expect(replacementPath.isValidUTF8) + + // Test that it can be used in path operations + let basePath = BinaryPath(string: "/tmp") + let fullPath = basePath.appending(replacementPath) + #expect(fullPath.stringValue == "/tmp/DIR�") + } + + @Test func AccentedCharacterPaths() { + // Test various accented characters + let accentedPaths = [ + "Foldèr with éncodïng", + "Évelyne Mère.txt", + "càraçt#èrë", + "épêcial", + ] + + for pathString in accentedPaths { + let path = BinaryPath(string: pathString) + #expect(path.stringValue == pathString) + #expect(path.isValidUTF8) + + // Test in a full path context + let fullPath = BinaryPath(string: "/home/user/\(pathString)") + #expect(fullPath.lastPathComponent.stringValue == pathString) + } + } + + @Test func SpecialCharacterPathComparison() { + // Test that paths with special characters can be compared and used in sets + let path1 = BinaryPath(string: "Répertoire (@vec) {càraçt#èrë} $épêcial") + let path2 = BinaryPath(string: "Répertoire (@vec) {càraçt#èrë} $épêcial") + let path3 = BinaryPath(string: "이루마 YIRUMA - River Flows in You.mp3") + + #expect(path1 == path2) + #expect(path1 != path3) + + var pathSet = Set() + pathSet.insert(path1) + pathSet.insert(path2) + pathSet.insert(path3) + + #expect(pathSet.count == 2) // path1 and path2 are equal + #expect(pathSet.contains(path1)) + #expect(pathSet.contains(path3)) + } + + @Test func SpecialCharacterPathCoding() throws { + // Test that special character paths can be encoded/decoded + let specialPaths = [ + "Répertoire (@vec) {càraçt#èrë} $épêcial", + "이루마 YIRUMA - River Flows in You.mp3", + "test\\test", + "DIR�", + ] + + let encoder = JSONEncoder() + let decoder = JSONDecoder() + + for pathString in specialPaths { + let original = BinaryPath(string: pathString) + let data = try encoder.encode(original) + let decoded = try decoder.decode(BinaryPath.self, from: data) + + #expect(original == decoded, "Coding round-trip failed for: \(pathString)") + #expect(decoded.stringValue == pathString, "String value not preserved for: \(pathString)") + } + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildSnapshotterTests/DiffKeyUnitTests.swift b/Tests/NativeBuilderTests/ContainerBuildSnapshotterTests/DiffKeyUnitTests.swift new file mode 100644 index 00000000..ec31dc42 --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildSnapshotterTests/DiffKeyUnitTests.swift @@ -0,0 +1,323 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerBuildIR +import Foundation +import Testing + +@testable import ContainerBuildSnapshotter + +/// Pure unit tests for DiffKey that don't require DirectoryDiffer or filesystem operations. +/// These tests use mock Diff data to test DiffKey.computeFromDiffs() in isolation. +@Suite struct DiffKeyUnitTests { + + // MARK: - Mock Data Helpers + + /// Create a mock Added diff entry + private func mockAdded( + path: String, + node: Diff.Modified.Node = .regular, + permissions: FilePermissions? = FilePermissions(rawValue: 0o644), + size: Int64? = 100, + uid: UInt32? = 1000, + gid: UInt32? = 1000, + xattrs: [String: Data]? = nil + ) -> Diff { + .added( + .init( + path: BinaryPath(string: path), + node: node, + permissions: permissions, + size: size, + modificationTime: Date(timeIntervalSince1970: 1_000_000), + linkTarget: nil, + uid: uid, + gid: gid, + xattrs: xattrs, + devMajor: nil, + devMinor: nil, + nlink: nil + ) + ) + } + + /// Create a mock Modified diff entry + private func mockModified( + path: String, + kind: Diff.Modified.Kind = .contentChanged, + node: Diff.Modified.Node = .regular, + permissions: FilePermissions? = FilePermissions(rawValue: 0o644), + size: Int64? = 100, + uid: UInt32? = 1000, + gid: UInt32? = 1000, + xattrs: [String: Data]? = nil + ) -> Diff { + .modified( + .init( + path: BinaryPath(string: path), + kind: kind, + node: node, + permissions: permissions, + size: size, + modificationTime: Date(timeIntervalSince1970: 1_000_000), + linkTarget: nil, + uid: uid, + gid: gid, + xattrs: xattrs, + devMajor: nil, + devMinor: nil, + nlink: nil + ) + ) + } + + /// Create a mock Deleted diff entry + private func mockDeleted(path: String) -> Diff { + .deleted(path: BinaryPath(string: path)) + } + + // MARK: - Tests + + @Test func emptyDiffProducesConsistentKey() async throws { + // Empty diffs should produce a consistent key + let key1 = try await DiffKey.computeFromDiffs( + [], + targetMount: URL(fileURLWithPath: "/tmp") + ) + let key2 = try await DiffKey.computeFromDiffs( + [], + targetMount: URL(fileURLWithPath: "/tmp") + ) + + #expect(key1 == key2) + #expect(key1.stringValue.hasPrefix("sha256:")) + } + + @Test func singleAddedFileProducesKey() async throws { + let diffs = [mockAdded(path: "file.txt")] + + let key = try await DiffKey.computeFromDiffs( + diffs, + targetMount: URL(fileURLWithPath: "/tmp") + ) + + #expect(key.stringValue.hasPrefix("sha256:")) + #expect(key.rawHex.count == 64) + } + + @Test func orderIndependentForSameChanges() async throws { + // Same changes in different order should produce the same key + let diffs1 = [ + mockAdded(path: "a.txt"), + mockAdded(path: "b.txt"), + mockAdded(path: "c.txt"), + ] + + let diffs2 = [ + mockAdded(path: "c.txt"), + mockAdded(path: "a.txt"), + mockAdded(path: "b.txt"), + ] + + let key1 = try await DiffKey.computeFromDiffs( + diffs1, + targetMount: URL(fileURLWithPath: "/tmp") + ) + let key2 = try await DiffKey.computeFromDiffs( + diffs2, + targetMount: URL(fileURLWithPath: "/tmp") + ) + + #expect(key1 == key2) + } + + @Test func differentOperationTypesProduceDifferentKeys() async throws { + let added = [mockAdded(path: "file.txt")] + let modified = [mockModified(path: "file.txt")] + let deleted = [mockDeleted(path: "file.txt")] + + let keyAdded = try await DiffKey.computeFromDiffs( + added, + targetMount: URL(fileURLWithPath: "/tmp") + ) + let keyModified = try await DiffKey.computeFromDiffs( + modified, + targetMount: URL(fileURLWithPath: "/tmp") + ) + let keyDeleted = try await DiffKey.computeFromDiffs( + deleted, + targetMount: URL(fileURLWithPath: "/tmp") + ) + + #expect(keyAdded != keyModified) + #expect(keyModified != keyDeleted) + #expect(keyAdded != keyDeleted) + } + + @Test func differentModificationKindsProduceDifferentKeys() async throws { + let contentChange = [mockModified(path: "file.txt", kind: .contentChanged)] + let metadataOnly = [mockModified(path: "file.txt", kind: .metadataOnly)] + let typeChange = [mockModified(path: "file.txt", kind: .typeChanged)] + + let keyContent = try await DiffKey.computeFromDiffs( + contentChange, + targetMount: URL(fileURLWithPath: "/tmp") + ) + let keyMetadata = try await DiffKey.computeFromDiffs( + metadataOnly, + targetMount: URL(fileURLWithPath: "/tmp") + ) + let keyType = try await DiffKey.computeFromDiffs( + typeChange, + targetMount: URL(fileURLWithPath: "/tmp") + ) + + #expect(keyContent != keyMetadata) + #expect(keyMetadata != keyType) + #expect(keyContent != keyType) + } + + @Test func differentPermissionsProduceDifferentKeys() async throws { + let perm644 = [mockAdded(path: "file.txt", permissions: FilePermissions(rawValue: 0o644))] + let perm755 = [mockAdded(path: "file.txt", permissions: FilePermissions(rawValue: 0o755))] + + let key644 = try await DiffKey.computeFromDiffs( + perm644, + targetMount: URL(fileURLWithPath: "/tmp") + ) + let key755 = try await DiffKey.computeFromDiffs( + perm755, + targetMount: URL(fileURLWithPath: "/tmp") + ) + + #expect(key644 != key755) + } + + @Test func xattrsAffectKey() async throws { + let noXattrs = [mockAdded(path: "file.txt")] + let withXattrs = [ + mockAdded( + path: "file.txt", + xattrs: ["user.test": Data("value".utf8)] + ) + ] + + let keyNoXattrs = try await DiffKey.computeFromDiffs( + noXattrs, + targetMount: URL(fileURLWithPath: "/tmp") + ) + let keyWithXattrs = try await DiffKey.computeFromDiffs( + withXattrs, + targetMount: URL(fileURLWithPath: "/tmp") + ) + + #expect(keyNoXattrs != keyWithXattrs) + } + + @Test func baseDigestAffectsKey() async throws { + let diffs = [mockAdded(path: "file.txt")] + let base1 = try Digest.compute(Data("base1".utf8), using: .sha256) + let base2 = try Digest.compute(Data("base2".utf8), using: .sha256) + + let keyNoBase = try await DiffKey.computeFromDiffs( + diffs, + baseDigest: nil, + targetMount: URL(fileURLWithPath: "/tmp") + ) + let keyBase1 = try await DiffKey.computeFromDiffs( + diffs, + baseDigest: base1, + targetMount: URL(fileURLWithPath: "/tmp") + ) + let keyBase2 = try await DiffKey.computeFromDiffs( + diffs, + baseDigest: base2, + targetMount: URL(fileURLWithPath: "/tmp") + ) + + #expect(keyNoBase != keyBase1) + #expect(keyBase1 != keyBase2) + #expect(keyNoBase != keyBase2) + } + + @Test func coupleToBaseParameter() async throws { + let diffs = [mockAdded(path: "file.txt")] + let base = try Digest.compute(Data("base".utf8), using: .sha256) + + let keyCoupled = try await DiffKey.computeFromDiffs( + diffs, + baseDigest: base, + targetMount: URL(fileURLWithPath: "/tmp"), + coupleToBase: true + ) + let keyUncoupled = try await DiffKey.computeFromDiffs( + diffs, + baseDigest: base, + targetMount: URL(fileURLWithPath: "/tmp"), + coupleToBase: false + ) + + #expect(keyCoupled != keyUncoupled) + } + + @Test func complexDiffSetProducesConsistentKey() async throws { + let diffs = [ + mockAdded(path: "new/file1.txt"), + mockAdded(path: "new/file2.txt", node: .directory), + mockModified(path: "existing/file.txt", kind: .contentChanged), + mockModified(path: "existing/dir", kind: .metadataOnly, node: .directory), + mockDeleted(path: "old/file.txt"), + mockDeleted(path: "old/dir"), + mockAdded(path: "link", node: .symlink), + ] + + let key1 = try await DiffKey.computeFromDiffs( + diffs, + targetMount: URL(fileURLWithPath: "/tmp") + ) + let key2 = try await DiffKey.computeFromDiffs( + diffs, + targetMount: URL(fileURLWithPath: "/tmp") + ) + + #expect(key1 == key2) + } + + @Test func socketsAndDevicesAreExcluded() async throws { + // Sockets and device nodes should be excluded from the key + let withSocketAndDevice = [ + mockAdded(path: "file.txt"), + mockAdded(path: "socket", node: .socket), + mockAdded(path: "device", node: .device), + ] + + let withoutSocketAndDevice = [ + mockAdded(path: "file.txt") + ] + + let key1 = try await DiffKey.computeFromDiffs( + withSocketAndDevice, + targetMount: URL(fileURLWithPath: "/tmp") + ) + let key2 = try await DiffKey.computeFromDiffs( + withoutSocketAndDevice, + targetMount: URL(fileURLWithPath: "/tmp") + ) + + // Keys should be the same since sockets and devices are excluded + #expect(key1 == key2) + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildSnapshotterTests/FileContentDifferTests.swift b/Tests/NativeBuilderTests/ContainerBuildSnapshotterTests/FileContentDifferTests.swift new file mode 100644 index 00000000..2a8ef205 --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildSnapshotterTests/FileContentDifferTests.swift @@ -0,0 +1,86 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation +import Testing + +@testable import ContainerBuildSnapshotter + +@Suite struct FileContentDifferTests { + + @Test func attributesOnlyShortCircuit() throws { + let d = FileContentDiffer() + let r = try d.diff(oldURL: nil, newURL: nil, attributesOnly: true) + #expect(r == .attributeOnly) + } + + @Test func nilSidesMeanContentChanged() throws { + let d = FileContentDiffer() + // old only + #expect(try d.diff(oldURL: URL(fileURLWithPath: "/tmp/missing"), newURL: nil) == .contentChanged) + // new only + #expect(try d.diff(oldURL: nil, newURL: URL(fileURLWithPath: "/tmp/missing")) == .contentChanged) + } + + @Test func equalContentAttributeOnly() throws { + try TestUtils.withTempDir { dir in + let a = dir.appendingPathComponent("a.txt") + let b = dir.appendingPathComponent("b.txt") + try TestUtils.writeString(a, "same") + try TestUtils.writeString(b, "same") + + let d = FileContentDiffer() + let r = try d.diff(oldURL: a, newURL: b) + #expect(r == .attributeOnly) + } + } + + @Test func changedContentDetected() throws { + try TestUtils.withTempDir { dir in + let a = dir.appendingPathComponent("a.txt") + let b = dir.appendingPathComponent("b.txt") + try TestUtils.writeString(a, "hello") + try TestUtils.writeString(b, "world") + + let d = FileContentDiffer() + let r = try d.diff(oldURL: a, newURL: b) + #expect(r == .contentChanged) + } + } + + @Test func largeFileChunking() throws { + try TestUtils.withTempDir { dir in + let a = dir.appendingPathComponent("a.bin") + let b = dir.appendingPathComponent("b.bin") + // ~10MB payload + let block = Data(repeating: 0xAB, count: 1024 * 1024) // 1MiB + var payload = Data() + for _ in 0..<10 { payload.append(block) } + try TestUtils.write(a, contents: payload) + try TestUtils.write(b, contents: payload) + + let d = FileContentDiffer() + #expect(try d.diff(oldURL: a, newURL: b) == .attributeOnly) + + // flip one byte + var changed = payload + changed[changed.count / 2] = 0xCD + try TestUtils.write(b, contents: changed) + + #expect(try d.diff(oldURL: a, newURL: b) == .contentChanged) + } + } +} diff --git a/Tests/NativeBuilderTests/ContainerBuildSnapshotterTests/TestUtils.swift b/Tests/NativeBuilderTests/ContainerBuildSnapshotterTests/TestUtils.swift new file mode 100644 index 00000000..e6b6d67c --- /dev/null +++ b/Tests/NativeBuilderTests/ContainerBuildSnapshotterTests/TestUtils.swift @@ -0,0 +1,82 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +// Simple temp-directory helper for tests +enum TestUtils { + static func withTempDir(_ body: (URL) throws -> T) throws -> T { + let dir = FileManager.default.temporaryDirectory.appendingPathComponent("cb-snap-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: dir) } + return try body(dir) + } + + static func withTempDirAsync(_ body: (URL) async throws -> T) async throws -> T { + let dir = FileManager.default.temporaryDirectory.appendingPathComponent("cb-snap-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: dir) } + return try await body(dir) + } + + @discardableResult + static func write(_ path: URL, contents: Data, permissions: UInt16? = nil) throws -> URL { + let fm = FileManager.default + try fm.createDirectory(at: path.deletingLastPathComponent(), withIntermediateDirectories: true) + fm.createFile(atPath: path.path, contents: contents) + if let mode = permissions { + try fm.setAttributes([.posixPermissions: NSNumber(value: mode)], ofItemAtPath: path.path) + } + return path + } + + @discardableResult + static func writeString(_ path: URL, _ string: String, permissions: UInt16? = nil) throws -> URL { + try write(path, contents: Data(string.utf8), permissions: permissions) + } + + static func readString(_ path: URL) throws -> String { + String(decoding: try Data(contentsOf: path), as: UTF8.self) + } + + static func makeSymlink(at: URL, to relativeTarget: String) throws { + let fm = FileManager.default + try fm.createDirectory(at: at.deletingLastPathComponent(), withIntermediateDirectories: true) + try fm.createSymbolicLink(atPath: at.path, withDestinationPath: relativeTarget) + } + + static func chmod(_ path: URL, mode: UInt16) throws { + try FileManager.default.setAttributes([.posixPermissions: NSNumber(value: mode)], ofItemAtPath: path.path) + } + + static func fileExists(_ path: URL) -> Bool { + FileManager.default.fileExists(atPath: path.path) + } + + static func mkdir(_ path: URL) throws { + try FileManager.default.createDirectory(at: path, withIntermediateDirectories: true) + } + + static func listAll(relativeTo root: URL) throws -> [String] { + guard let en = FileManager.default.enumerator(at: root, includingPropertiesForKeys: nil) else { return [] } + var out: [String] = [] + while let u = en.nextObject() as? URL { + let rel = u.path.replacingOccurrences(of: root.path + "/", with: "") + out.append(rel) + } + return out.sorted() + } +}