diff --git a/Package.swift b/Package.swift index f728e375..cd0cfa93 100644 --- a/Package.swift +++ b/Package.swift @@ -22,7 +22,7 @@ import PackageDescription let releaseVersion = ProcessInfo.processInfo.environment["RELEASE_VERSION"] ?? "0.0.0" let gitCommit = ProcessInfo.processInfo.environment["GIT_COMMIT"] ?? "unspecified" -let builderShimVersion = "0.7.0" +let builderShimVersion = "0.8.0" let scVersion = "0.25.0" let package = Package( diff --git a/Sources/ContainerBuild/BuildFSSync.swift b/Sources/ContainerBuild/BuildFSSync.swift index c26f8597..c258f22f 100644 --- a/Sources/ContainerBuild/BuildFSSync.swift +++ b/Sources/ContainerBuild/BuildFSSync.swift @@ -18,6 +18,7 @@ import Collections import ContainerAPIClient import ContainerizationArchive import ContainerizationOCI +import CryptoKit import Foundation import GRPC @@ -199,7 +200,7 @@ actor BuildFSSync: BuildPipelineHandler { format: .paxRestricted, filter: .none) - try Archiver.compress( + let tarHash = try Archiver.compress( source: contextDir, destination: tarURL, writerConfiguration: writerCfg @@ -229,6 +230,25 @@ actor BuildFSSync: BuildPipelineHandler { pathInArchive: URL(fileURLWithPath: rel)) } + let hash = tarHash.compactMap { String(format: "%02x", $0) }.joined() + let header = BuildTransfer( + id: packet.id, + source: tarURL.path, + complete: false, + isDir: false, + metadata: [ + "os": "linux", + "stage": "fssync", + "mode": "tar", + "hash": hash, + ] + ) + var resp = ClientStream() + resp.buildID = buildID + resp.buildTransfer = header + resp.packetType = .buildTransfer(header) + sender.yield(resp) + for try await chunk in try tarURL.bufferedCopyReader() { let part = BuildTransfer( id: packet.id, diff --git a/Sources/Services/ContainerAPIService/Client/Archiver.swift b/Sources/Services/ContainerAPIService/Client/Archiver.swift index d4b03972..d20fb632 100644 --- a/Sources/Services/ContainerAPIService/Client/Archiver.swift +++ b/Sources/Services/ContainerAPIService/Client/Archiver.swift @@ -16,10 +16,11 @@ import ContainerizationArchive import ContainerizationOS +import CryptoKit import Foundation public final class Archiver: Sendable { - public struct ArchiveEntryInfo: Sendable { + public struct ArchiveEntryInfo: Sendable, Codable { public let pathOnHost: URL public let pathInArchive: URL @@ -48,13 +49,15 @@ public final class Archiver: Sendable { followSymlinks: Bool = false, writerConfiguration: ArchiveWriterConfiguration = ArchiveWriterConfiguration(format: .paxRestricted, filter: .gzip), closure: (URL) -> ArchiveEntryInfo? - ) throws { + ) throws -> SHA256.Digest { let source = source.standardizedFileURL let destination = destination.standardizedFileURL let fileManager = FileManager.default try? fileManager.removeItem(at: destination) + var hasher = SHA256() + do { let directory = destination.deletingLastPathComponent() try fileManager.createDirectory(at: directory, withIntermediateDirectories: true) @@ -71,7 +74,8 @@ public final class Archiver: Sendable { entryInfo.append(info) } } else { - while let relPath = enumerator.nextObject() as? String { + let relPaths = enumerator.compactMap { $0 as? String } + for relPath in relPaths.sorted(by: { $0 < $1 }) { let url = source.appending(path: relPath).standardizedFileURL guard let info = closure(url) else { continue @@ -85,17 +89,23 @@ public final class Archiver: Sendable { ) try archiver.open(file: destination) + let encoder = JSONEncoder() + encoder.outputFormatting = .sortedKeys + for info in entryInfo { guard let entry = try Self._createEntry(entryInfo: info) else { throw Error.failedToCreateEntry } - try Self._compressFile(item: info.pathOnHost, entry: entry, archiver: archiver) + hasher.update(data: try encoder.encode(info)) + try Self._compressFile(item: info.pathOnHost, entry: entry, archiver: archiver, hasher: &hasher) } try archiver.finishEncoding() } catch { try? fileManager.removeItem(at: destination) throw error } + + return hasher.finalize() } public static func uncompress(source: URL, destination: URL) throws { @@ -186,7 +196,7 @@ public final class Archiver: Sendable { } // MARK: private functions - private static func _compressFile(item: URL, entry: WriteEntry, archiver: ArchiveWriter) throws { + private static func _compressFile(item: URL, entry: WriteEntry, archiver: ArchiveWriter, hasher: inout SHA256) throws { guard let stream = InputStream(url: item) else { return } @@ -204,6 +214,7 @@ public final class Archiver: Sendable { break } else { let data = Data(bytes: readBuffer, count: byteRead) + hasher.update(data: data) try data.withUnsafeBytes { pointer in try writer.writeChunk(data: pointer) } diff --git a/Tests/CLITests/Subcommands/Build/CLIBuilderTest.swift b/Tests/CLITests/Subcommands/Build/CLIBuilderTest.swift index 74c23abf..0269f0ed 100644 --- a/Tests/CLITests/Subcommands/Build/CLIBuilderTest.swift +++ b/Tests/CLITests/Subcommands/Build/CLIBuilderTest.swift @@ -457,6 +457,59 @@ extension TestCLIBuildBase { #expect(try self.inspectImage(tag3) == tag3, "expected to have successfully built \(tag3)") } + @Test func testBuildAfterContextChange() throws { + let name = "test-build-context-change" + let tempDir: URL = try createTempDir() + + // Create initial context with file "foo" containing "initial" + let dockerfile = + """ + FROM ghcr.io/linuxcontainers/alpine:3.20 + COPY foo /foo + COPY bar /bar + """ + let initialContent = "initial".data(using: .utf8)! + let context: [FileSystemEntry] = [ + .file("foo", content: .data(Data((0..<4 * 1024 * 1024).map { UInt8($0 % 256) }))), + .file("bar", content: .data(initialContent)), + ] + try createContext(tempDir: tempDir, dockerfile: dockerfile, context: context) + + // Build first image + let imageName1 = "\(name):v1" + let containerName1 = "\(name)-container-v1" + try self.build(tag: imageName1, tempDir: tempDir) + #expect(try self.inspectImage(imageName1) == imageName1, "expected to have successfully built \(imageName1)") + + // Run container and verify content is "initial" + try self.doLongRun(name: containerName1, image: imageName1) + defer { + try? self.doStop(name: containerName1) + } + var output = try doExec(name: containerName1, cmd: ["cat", "/bar"]) + #expect(output == "initial", "expected file contents to be 'initial', instead got '\(output)'") + + // Update the file "foo" to contain "updated" + let updatedContent = "updated".data(using: .utf8)! + let contextDir = tempDir.appendingPathComponent("context") + let barPath = contextDir.appendingPathComponent("bar") + try updatedContent.write(to: barPath, options: .atomic) + + // Build second image + let imageName2 = "\(name):v2" + let containerName2 = "\(name)-container-v2" + try self.build(tag: imageName2, tempDir: tempDir) + #expect(try self.inspectImage(imageName2) == imageName2, "expected to have successfully built \(imageName2)") + + // Run container and verify content is "updated" + try self.doLongRun(name: containerName2, image: imageName2) + defer { + try? self.doStop(name: containerName2) + } + output = try doExec(name: containerName2, cmd: ["cat", "/bar"]) + #expect(output == "updated", "expected file contents to be 'updated', instead got '\(output)'") + } + @Test func testBuildWithDockerfileFromStdin() throws { let tempDir: URL = try createTempDir() let dockerfile =