From fabfc555688863fcc65029334289dd71afbd08f9 Mon Sep 17 00:00:00 2001 From: jwhur <57657645+JaewonHur@users.noreply.github.com> Date: Thu, 5 Feb 2026 09:53:31 -0800 Subject: [PATCH] Send tar hash in the first BuildTransfer packet (#1149) Send the hash of entire tar file in the first BuildTransfer packet to prevent container-builder-shim from using stale cached contents. This PR resolves #1143. This PR relies on apple/container-builder-shim#64. ## Type of Change - [X] Bug fix - [ ] New feature - [ ] Breaking change - [ ] Documentation update ## Motivation and Context Current container-builder-shim uses only first few bytes of tar file as checksum, which leads to the usage of stale cached contents if the change of build context is not included in the first bytes of tar file. ## Testing - [X] Tested locally - [ ] Added/updated tests - [ ] Added/updated docs --------- Co-authored-by: Ronit Sabhaya Co-authored-by: J Logan --- Package.swift | 2 +- Sources/ContainerBuild/BuildFSSync.swift | 22 +++++++- .../ContainerAPIService/Client/Archiver.swift | 21 ++++++-- .../Subcommands/Build/CLIBuilderTest.swift | 53 +++++++++++++++++++ 4 files changed, 91 insertions(+), 7 deletions(-) diff --git a/Package.swift b/Package.swift index f728e375..cd0cfa93 100644 --- a/Package.swift +++ b/Package.swift @@ -22,7 +22,7 @@ import PackageDescription let releaseVersion = ProcessInfo.processInfo.environment["RELEASE_VERSION"] ?? "0.0.0" let gitCommit = ProcessInfo.processInfo.environment["GIT_COMMIT"] ?? "unspecified" -let builderShimVersion = "0.7.0" +let builderShimVersion = "0.8.0" let scVersion = "0.25.0" let package = Package( diff --git a/Sources/ContainerBuild/BuildFSSync.swift b/Sources/ContainerBuild/BuildFSSync.swift index c26f8597..c258f22f 100644 --- a/Sources/ContainerBuild/BuildFSSync.swift +++ b/Sources/ContainerBuild/BuildFSSync.swift @@ -18,6 +18,7 @@ import Collections import ContainerAPIClient import ContainerizationArchive import ContainerizationOCI +import CryptoKit import Foundation import GRPC @@ -199,7 +200,7 @@ actor BuildFSSync: BuildPipelineHandler { format: .paxRestricted, filter: .none) - try Archiver.compress( + let tarHash = try Archiver.compress( source: contextDir, destination: tarURL, writerConfiguration: writerCfg @@ -229,6 +230,25 @@ actor BuildFSSync: BuildPipelineHandler { pathInArchive: URL(fileURLWithPath: rel)) } + let hash = tarHash.compactMap { String(format: "%02x", $0) }.joined() + let header = BuildTransfer( + id: packet.id, + source: tarURL.path, + complete: false, + isDir: false, + metadata: [ + "os": "linux", + "stage": "fssync", + "mode": "tar", + "hash": hash, + ] + ) + var resp = ClientStream() + resp.buildID = buildID + resp.buildTransfer = header + resp.packetType = .buildTransfer(header) + sender.yield(resp) + for try await chunk in try tarURL.bufferedCopyReader() { let part = BuildTransfer( id: packet.id, diff --git a/Sources/Services/ContainerAPIService/Client/Archiver.swift b/Sources/Services/ContainerAPIService/Client/Archiver.swift index d4b03972..d20fb632 100644 --- a/Sources/Services/ContainerAPIService/Client/Archiver.swift +++ b/Sources/Services/ContainerAPIService/Client/Archiver.swift @@ -16,10 +16,11 @@ import ContainerizationArchive import ContainerizationOS +import CryptoKit import Foundation public final class Archiver: Sendable { - public struct ArchiveEntryInfo: Sendable { + public struct ArchiveEntryInfo: Sendable, Codable { public let pathOnHost: URL public let pathInArchive: URL @@ -48,13 +49,15 @@ public final class Archiver: Sendable { followSymlinks: Bool = false, writerConfiguration: ArchiveWriterConfiguration = ArchiveWriterConfiguration(format: .paxRestricted, filter: .gzip), closure: (URL) -> ArchiveEntryInfo? - ) throws { + ) throws -> SHA256.Digest { let source = source.standardizedFileURL let destination = destination.standardizedFileURL let fileManager = FileManager.default try? fileManager.removeItem(at: destination) + var hasher = SHA256() + do { let directory = destination.deletingLastPathComponent() try fileManager.createDirectory(at: directory, withIntermediateDirectories: true) @@ -71,7 +74,8 @@ public final class Archiver: Sendable { entryInfo.append(info) } } else { - while let relPath = enumerator.nextObject() as? String { + let relPaths = enumerator.compactMap { $0 as? String } + for relPath in relPaths.sorted(by: { $0 < $1 }) { let url = source.appending(path: relPath).standardizedFileURL guard let info = closure(url) else { continue @@ -85,17 +89,23 @@ public final class Archiver: Sendable { ) try archiver.open(file: destination) + let encoder = JSONEncoder() + encoder.outputFormatting = .sortedKeys + for info in entryInfo { guard let entry = try Self._createEntry(entryInfo: info) else { throw Error.failedToCreateEntry } - try Self._compressFile(item: info.pathOnHost, entry: entry, archiver: archiver) + hasher.update(data: try encoder.encode(info)) + try Self._compressFile(item: info.pathOnHost, entry: entry, archiver: archiver, hasher: &hasher) } try archiver.finishEncoding() } catch { try? fileManager.removeItem(at: destination) throw error } + + return hasher.finalize() } public static func uncompress(source: URL, destination: URL) throws { @@ -186,7 +196,7 @@ public final class Archiver: Sendable { } // MARK: private functions - private static func _compressFile(item: URL, entry: WriteEntry, archiver: ArchiveWriter) throws { + private static func _compressFile(item: URL, entry: WriteEntry, archiver: ArchiveWriter, hasher: inout SHA256) throws { guard let stream = InputStream(url: item) else { return } @@ -204,6 +214,7 @@ public final class Archiver: Sendable { break } else { let data = Data(bytes: readBuffer, count: byteRead) + hasher.update(data: data) try data.withUnsafeBytes { pointer in try writer.writeChunk(data: pointer) } diff --git a/Tests/CLITests/Subcommands/Build/CLIBuilderTest.swift b/Tests/CLITests/Subcommands/Build/CLIBuilderTest.swift index 74c23abf..0269f0ed 100644 --- a/Tests/CLITests/Subcommands/Build/CLIBuilderTest.swift +++ b/Tests/CLITests/Subcommands/Build/CLIBuilderTest.swift @@ -457,6 +457,59 @@ extension TestCLIBuildBase { #expect(try self.inspectImage(tag3) == tag3, "expected to have successfully built \(tag3)") } + @Test func testBuildAfterContextChange() throws { + let name = "test-build-context-change" + let tempDir: URL = try createTempDir() + + // Create initial context with file "foo" containing "initial" + let dockerfile = + """ + FROM ghcr.io/linuxcontainers/alpine:3.20 + COPY foo /foo + COPY bar /bar + """ + let initialContent = "initial".data(using: .utf8)! + let context: [FileSystemEntry] = [ + .file("foo", content: .data(Data((0..<4 * 1024 * 1024).map { UInt8($0 % 256) }))), + .file("bar", content: .data(initialContent)), + ] + try createContext(tempDir: tempDir, dockerfile: dockerfile, context: context) + + // Build first image + let imageName1 = "\(name):v1" + let containerName1 = "\(name)-container-v1" + try self.build(tag: imageName1, tempDir: tempDir) + #expect(try self.inspectImage(imageName1) == imageName1, "expected to have successfully built \(imageName1)") + + // Run container and verify content is "initial" + try self.doLongRun(name: containerName1, image: imageName1) + defer { + try? self.doStop(name: containerName1) + } + var output = try doExec(name: containerName1, cmd: ["cat", "/bar"]) + #expect(output == "initial", "expected file contents to be 'initial', instead got '\(output)'") + + // Update the file "foo" to contain "updated" + let updatedContent = "updated".data(using: .utf8)! + let contextDir = tempDir.appendingPathComponent("context") + let barPath = contextDir.appendingPathComponent("bar") + try updatedContent.write(to: barPath, options: .atomic) + + // Build second image + let imageName2 = "\(name):v2" + let containerName2 = "\(name)-container-v2" + try self.build(tag: imageName2, tempDir: tempDir) + #expect(try self.inspectImage(imageName2) == imageName2, "expected to have successfully built \(imageName2)") + + // Run container and verify content is "updated" + try self.doLongRun(name: containerName2, image: imageName2) + defer { + try? self.doStop(name: containerName2) + } + output = try doExec(name: containerName2, cmd: ["cat", "/bar"]) + #expect(output == "updated", "expected file contents to be 'updated', instead got '\(output)'") + } + @Test func testBuildWithDockerfileFromStdin() throws { let tempDir: URL = try createTempDir() let dockerfile =