Files
container/Sources/ContainerBuild/BuildFSSync.swift
T
J Logan ca4884f8f3 Build context fix, bump to containerization 0.52.1. (#2415)
- `BuildFSSync.read()` and `info()` resolved a 
  requested path, checked that the result was
  inside the build context, and then opened the
  original path. A symlink replaced between the
  check and the open was followed to its new
  target, which could be outside the context.
-This change opens each requested file by
  walking its path one component at a time from
  a directory descriptor pinned on the context
  root, using `FileDescriptorOps` from
  `containerization`. Symlinks are followed only
  when the target is relative and stays beneath
  the context root. The metadata and the content
  returned to the builder both come from the
  same open descriptor.
- Behavior change: a symlink with an absolute
  target is now rejected by `read()` and `info()`,
  even when the target is inside the context.
  Relative symlinks that stay inside the context
  still work. `walk()`-based transfers are unchanged.
- Dependency: Requires `containerization` 0.50.0
  for its `FileDescriptorOps` API enhancements.
  This change updates the library from 0.52.0
  to 0.52.1.
2026-10-19 14:14:06 -07:00

618 lines
24 KiB
Swift

//===----------------------------------------------------------------------===//
// Copyright © 2025-2026 Apple Inc. and the container project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import Collections
import ContainerAPIClient
import ContainerizationArchive
import ContainerizationOCI
import ContainerizationOS
import CryptoKit
import Foundation
import GRPCCore
import SystemPackage
/// Handles the `fssync` stage of the build protocol.
///
/// When BuildKit needs build-context files it sends `Walk`, `Read`, and `Info`
/// requests to the shim, which proxies them over the gRPC stream to this actor.
///
/// ## Primary path: Walk (tar mode)
///
/// `Walk` is the primary data path. The host packs all requested context paths
/// into a tar archive and streams it to the shim. The shim unpacks the tar to a
/// local cache and presents the files to BuildKit via `DiffCopy`. BuildKit then
/// issues `PACKET_REQ` for regular files it needs; the shim serves those from
/// the local cache without any further calls to the host.
///
/// When a context path is a symlink whose target lies within the context root,
/// ``walk(_:_:_:)`` adds the target to the archive alongside the symlink so
/// BuildKit can dereference it during `COPY`/`ADD` processing.
///
/// ## Fallback path: Info + Read
///
/// `FS.Open()` in the shim falls back to `Info` followed by `Read` calls when
/// its local checksum cache is unpopulated (a narrow race window at the start of
/// a build). These paths are not exercised during a normal build.
///
/// ## Symlink safety
///
/// The host enforces that no file served to the builder resolves to a path
/// outside the context root. If any component of a requested path is a symlink
/// whose target lies outside the context root the request is rejected.
///
/// ``read(_:_:_:)`` and ``info(_:_:_:)`` enforce this by walking the path
/// component by component from a directory descriptor pinned on the context
/// root (``FileDescriptorOps``), so there is no window between checking a path
/// and using it in which a symlink can be swapped in. Symlinks are followed
/// only to relative targets that stay beneath the context root.
/// Dockerignore filtering is **not** applied here; the shim applies it after
/// unpacking the tar.
actor BuildFSSync: BuildPipelineHandler {
let contextDir: URL
/// Descriptor for `contextDir`, opened once so every request is resolved
/// relative to the same directory regardless of later changes to its path.
private let contextFD: FileDescriptor
init(_ contextDir: URL) throws {
let resolved = contextDir.resolvingSymlinksInPath()
guard FileManager.default.fileExists(atPath: contextDir.cleanPath) else {
throw Error.contextNotFound(contextDir.cleanPath)
}
guard resolved.isDirectory else {
throw Error.contextIsNotDirectory(contextDir.cleanPath)
}
self.contextDir = resolved
self.contextFD = try FileDescriptor.open(
FilePath(resolved.cleanPath), .readOnly, options: [.directory, .closeOnExec])
}
deinit {
try? contextFD.close()
}
nonisolated func accept(_ packet: ServerStream) throws -> Bool {
guard let buildTransfer = packet.getBuildTransfer() else {
return false
}
guard buildTransfer.stage() == "fssync" else {
return false
}
return true
}
func handle(_ sender: AsyncStream<ClientStream>.Continuation, _ packet: ServerStream) async throws {
guard let buildTransfer = packet.getBuildTransfer() else {
throw Error.buildTransferMissing
}
guard let method = buildTransfer.method() else {
throw Error.methodMissing
}
switch try FSSyncMethod(method) {
case .read:
try await self.read(sender, buildTransfer, packet.buildID)
case .info:
try await self.info(sender, buildTransfer, packet.buildID)
case .walk:
try await self.walk(sender, buildTransfer, packet.buildID)
}
}
/// Serves the content of a single context file to the shim.
///
/// Called only via the shim's `FS.Open()` fallback path, not during a
/// normal `Walk`-based build. The file is opened by walking its path from
/// the pinned context descriptor, following only symlinks that stay beneath
/// the context root; anything else is rejected with ``Error/pathIsNotChild``.
func read(_ sender: AsyncStream<ClientStream>.Continuation, _ packet: BuildTransfer, _ buildID: String) async throws {
let offset: UInt64 = packet.offset() ?? 0
let size: Int = packet.len() ?? 0
let relative = try self.relativePath(for: packet.source)
let (data, status) = try self.confined(packet.source) { () -> (Data, FileDescriptorOps.FileStatus) in
do {
let fd = try FileDescriptorOps.openFile(self.contextFD, relativePath: relative, symlinks: .followBeneath)
defer { try? fd.close() }
// Metadata and content both come from the descriptor, so they
// describe the same inode.
let status = try FileDescriptorOps.status(of: fd)
let handle = FileHandle(fileDescriptor: fd.rawValue, closeOnDealloc: false)
try handle.seek(toOffset: offset)
let data = size == 0 ? try handle.readToEnd() : try handle.read(upToCount: size)
return (data ?? Data(), status)
} catch FileDescriptorOps.Error.conflict(.directory) {
return (Data(), try self.entryStatus(relative))
}
}
let transfer = BuildTransfer(id: packet.id, source: relative.string, status: status, complete: true, data: data)
var response = ClientStream()
response.buildID = buildID
response.buildTransfer = transfer
response.packetType = .buildTransfer(transfer)
sender.yield(response)
}
/// Returns metadata (mode, size, modification time, uid/gid) for a single
/// context path.
///
/// Called only via the shim's `FS.Open()` fallback path, not during a
/// normal `Walk`-based build. Applies the same descriptor-relative
/// confinement as ``read(_:_:_:)``. The metadata is that of the entry
/// itself (a symlink is not dereferenced), but a symlink is rejected unless
/// it also resolves beneath the context root.
func info(_ sender: AsyncStream<ClientStream>.Continuation, _ packet: BuildTransfer, _ buildID: String) async throws {
let relative = try self.relativePath(for: packet.source)
let status = try self.confined(packet.source) { () -> FileDescriptorOps.FileStatus in
let status = try self.entryStatus(relative)
if status.type == .symlink {
do {
try FileDescriptorOps.openFile(self.contextFD, relativePath: relative, symlinks: .followBeneath).close()
} catch FileDescriptorOps.Error.conflict, FileDescriptorOps.Error.notFound {
// The link stays beneath the context; its target is just not a regular file.
}
}
return status
}
let transfer = BuildTransfer(id: packet.id, source: relative.string, status: status, complete: true)
var response = ClientStream()
response.buildID = buildID
response.buildTransfer = transfer
response.packetType = .buildTransfer(transfer)
sender.yield(response)
}
/// Converts a request `source` into a path relative to the context root.
///
/// Absolute sources inside the context are made relative to it; other
/// absolute sources are taken to be relative to the context root, as the
/// shim sometimes sends them. `..` is collapsed lexically and a result
/// that leaves the context is rejected. This is only normalisation: the
/// real confinement is done by the descriptor walk.
private func relativePath(for source: String) throws -> FilePath {
if source.hasPrefix("/") {
let url = URL(fileURLWithPath: source).standardizedFileURL
if let relative = try? url.relativeChildPath(to: self.contextDir) {
return FilePath(relative)
}
let fallback = self.contextDir.appending(components: source.cleanPathComponent).standardizedFileURL
return FilePath(try fallback.relativeChildPath(to: self.contextDir))
}
let url = self.contextDir.appendingPathComponent(source).standardizedFileURL
return FilePath(try url.relativeChildPath(to: self.contextDir))
}
/// Metadata of the entry at `relative` without dereferencing it.
private func entryStatus(_ relative: FilePath) throws -> FileDescriptorOps.FileStatus {
guard let status = try FileDescriptorOps.status(self.contextFD, relativePath: relative, symlinks: .followBeneath) else {
throw FileDescriptorOps.Error.notFound
}
return status
}
/// Runs `body`, reporting a refused symlink or path as ``Error/pathIsNotChild``.
private func confined<T>(_ source: String, _ body: () throws -> T) throws -> T {
do {
return try body()
} catch FileDescriptorOps.Error.cannotFollowSymlink, FileDescriptorOps.Error.invalidRelativePath {
throw Error.pathIsNotChild(source, self.contextDir.cleanPath)
}
}
private struct DirEntry: Hashable {
let url: URL
let isDirectory: Bool
let relativePath: String
func hash(into hasher: inout Hasher) {
hasher.combine(relativePath)
}
static func == (lhs: DirEntry, rhs: DirEntry) -> Bool {
lhs.relativePath == rhs.relativePath
}
}
/// Packs requested context paths into a tar archive and streams it to the shim.
///
/// This is the primary data path for build-context transfer. BuildKit sends
/// a `Walk` request whose `followpaths` field names the context paths needed
/// for the current build step (e.g. the source of a `COPY` instruction).
/// The host resolves those globs, builds an entry set, and passes it to
/// `Archiver.compress` to produce the tar.
///
/// For any symlink in the entry set whose target lies within the context
/// root, the target is added to the entry set so BuildKit can dereference
/// the symlink during `COPY`/`ADD` processing without a separate request.
/// Symlinks whose targets lie outside the context root are included as
/// symlink entries but their targets are not; BuildKit will resolve them
/// against the shim's local filesystem on Linux, not the macOS host.
///
/// Dockerignore filtering is the shim's responsibility and is applied after
/// the tar is unpacked; this method has no knowledge of `.dockerignore`.
func walk(
_ sender: AsyncStream<ClientStream>.Continuation,
_ packet: BuildTransfer,
_ buildID: String
) async throws {
let wantsTar = packet.mode() == "tar"
var entries: [String: Set<DirEntry>] = [:]
let followPaths: [String] = packet.followPaths() ?? []
let followPathsWalked = try walk(root: self.contextDir, includePatterns: followPaths)
for url in followPathsWalked {
guard self.contextDir.absoluteURL.cleanPath != url.absoluteURL.cleanPath else {
continue
}
guard self.contextDir.parentOf(url) else {
continue
}
let relPath = try url.relativeChildPath(to: contextDir)
let parentPath = try url.deletingLastPathComponent().relativeChildPath(to: contextDir)
let entry = DirEntry(url: url, isDirectory: url.hasDirectoryPath, relativePath: relPath)
entries[parentPath, default: []].insert(entry)
if url.isSymlink {
let target: URL = url.resolvingSymlinksInPath()
if self.contextDir.parentOf(target) {
let relPath = try target.relativeChildPath(to: self.contextDir)
let entry = DirEntry(url: target, isDirectory: target.hasDirectoryPath, relativePath: relPath)
let parentPath: String = try target.deletingLastPathComponent().relativeChildPath(to: self.contextDir)
entries[parentPath, default: []].insert(entry)
}
}
}
var fileOrder = [String]()
try processDirectory("", inputEntries: entries, processedPaths: &fileOrder)
if !wantsTar {
let fileInfos = try fileOrder.map { rel -> FileInfo in
try FileInfo(path: contextDir.appendingPathComponent(rel), contextDir: contextDir)
}
let data = try JSONEncoder().encode(fileInfos)
let transfer = BuildTransfer(
id: packet.id,
source: packet.source,
complete: true,
isDir: false,
metadata: [
"os": "linux",
"stage": "fssync",
"mode": "json",
],
data: data
)
var resp = ClientStream()
resp.buildID = buildID
resp.buildTransfer = transfer
resp.packetType = .buildTransfer(transfer)
sender.yield(resp)
return
}
let tarURL = URL.temporaryDirectory
.appendingPathComponent(UUID().uuidString + ".tar")
defer { try? FileManager.default.removeItem(at: tarURL) }
let writerCfg = ArchiveWriterConfiguration(
format: .paxRestricted,
filter: .none)
let tarHash = try Archiver.compress(
source: contextDir,
destination: tarURL,
writerConfiguration: writerCfg
) { url in
guard let rel = try? url.relativeChildPath(to: contextDir) else {
return nil
}
guard let parent = try? url.deletingLastPathComponent().relativeChildPath(to: self.contextDir) else {
return nil
}
guard let items = entries[parent] else {
return nil
}
let include = items.contains { item in
item.relativePath == rel
}
guard include else {
return nil
}
return Archiver.ArchiveEntryInfo(
pathOnHost: url,
pathInArchive: URL(fileURLWithPath: rel))
}
let hash = tarHash.compactMap { String(format: "%02x", $0) }.joined()
let header = BuildTransfer(
id: packet.id,
source: tarURL.path,
complete: false,
isDir: false,
metadata: [
"os": "linux",
"stage": "fssync",
"mode": "tar",
"hash": hash,
]
)
var resp = ClientStream()
resp.buildID = buildID
resp.buildTransfer = header
resp.packetType = .buildTransfer(header)
sender.yield(resp)
for try await chunk in try tarURL.bufferedCopyReader() {
let part = BuildTransfer(
id: packet.id,
source: tarURL.path,
complete: false,
isDir: false,
metadata: [
"os": "linux",
"stage": "fssync",
"mode": "tar",
],
data: chunk
)
var resp = ClientStream()
resp.buildID = buildID
resp.buildTransfer = part
resp.packetType = .buildTransfer(part)
sender.yield(resp)
}
let done = BuildTransfer(
id: packet.id,
source: tarURL.path,
complete: true,
isDir: false,
metadata: [
"os": "linux",
"stage": "fssync",
"mode": "tar",
],
data: Data()
)
var finalResp = ClientStream()
finalResp.buildID = buildID
finalResp.buildTransfer = done
finalResp.packetType = .buildTransfer(done)
sender.yield(finalResp)
}
func walk(root: URL, includePatterns: [String]) throws -> [URL] {
let globber = Globber(root)
for p in includePatterns {
try globber.match(p)
}
return Array(globber.results)
}
private func processDirectory(
_ currentDir: String,
inputEntries: [String: Set<DirEntry>],
processedPaths: inout [String]
) throws {
guard let entries = inputEntries[currentDir] else {
return
}
// Sort purely by lexicographical order of relativePath
let sortedEntries = entries.sorted { $0.relativePath < $1.relativePath }
for entry in sortedEntries {
processedPaths.append(entry.relativePath)
if entry.isDirectory {
try processDirectory(
entry.relativePath,
inputEntries: inputEntries,
processedPaths: &processedPaths
)
}
}
}
struct FileInfo: Codable {
let name: String
let modTime: String
let mode: UInt32
let size: UInt64
let isDir: Bool
let uid: UInt32
let gid: UInt32
let target: String
init(path: URL, contextDir: URL) throws {
// Always report the literal, unresolved on-disk symlink target —
// the same value tar mode provides via Archiver's use of
// destinationOfSymbolicLink — rather than a host-resolved path.
self.target = path.isSymlink ? try FileManager.default.destinationOfSymbolicLink(atPath: path.cleanPath) : ""
self.name = try path.relativeChildPath(to: contextDir)
self.modTime = try path.modTime()
self.mode = try path.mode()
self.size = try path.size()
self.isDir = path.hasDirectoryPath
self.uid = 0
self.gid = 0
}
}
enum FSSyncMethod: String {
case read = "Read"
case info = "Info"
case walk = "Walk"
init(_ method: String) throws {
switch method {
case "Read":
self = .read
case "Info":
self = .info
case "Walk":
self = .walk
default:
throw Error.unknownMethod(method)
}
}
}
}
extension BuildFSSync {
enum Error: Swift.Error, CustomStringConvertible, Equatable {
case buildTransferMissing
case methodMissing
case unknownMethod(String)
case contextNotFound(String)
case contextIsNotDirectory(String)
case couldNotDetermineFileSize(String)
case couldNotDetermineModTime(String)
case couldNotDetermineFileMode(String)
case invalidOffsetSizeForFile(String, UInt64, Int)
case couldNotDetermineUID(String)
case couldNotDetermineGID(String)
case pathIsNotChild(String, String)
var description: String {
switch self {
case .buildTransferMissing:
return "buildTransfer field missing in packet"
case .methodMissing:
return "method is missing in request"
case .unknownMethod(let m):
return "unknown content-store method \(m)"
case .contextNotFound(let path):
return "context dir \(path) not found"
case .contextIsNotDirectory(let path):
return "context \(path) not a directory"
case .couldNotDetermineFileSize(let path):
return "could not determine size of file \(path)"
case .couldNotDetermineModTime(let path):
return "could not determine last modified time of \(path)"
case .couldNotDetermineFileMode(let path):
return "could not determine posix permissions (FileMode) of \(path)"
case .invalidOffsetSizeForFile(let digest, let offset, let size):
return "invalid request for file: \(digest) with offset: \(offset) size: \(size)"
case .couldNotDetermineUID(let path):
return "could not determine UID of file at path: \(path)"
case .couldNotDetermineGID(let path):
return "could not determine GID of file at path: \(path)"
case .pathIsNotChild(let path, let parent):
return "\(path) is not a child of \(parent)"
}
}
}
}
extension BuildTransfer {
fileprivate init(id: String, source: String, complete: Bool, isDir: Bool, metadata: [String: String], data: Data? = nil) {
self.init()
self.id = id
self.source = source
self.direction = .outof
self.complete = complete
self.metadata = metadata
self.isDirectory = isDir
if let data {
self.data = data
}
}
}
extension BuildTransfer {
fileprivate init(id: String, source: String, status: FileDescriptorOps.FileStatus, complete: Bool, data: Data = Data()) {
self.init(
id: id,
source: source,
complete: complete,
isDir: status.type == .directory,
metadata: [
"os": "linux",
"stage": "fssync",
"mode": String(status.permissions.rawValue),
"size": String(status.size),
"modified_at": Date(timeIntervalSince1970: TimeInterval(status.modificationSeconds)).rfc3339(),
"uid": String(status.userID),
"gid": String(status.groupID),
],
data: data
)
}
}
extension URL {
fileprivate func size() throws -> UInt64 {
let attrs = try FileManager.default.attributesOfItem(atPath: self.cleanPath)
if let size = attrs[FileAttributeKey.size] as? UInt64 {
return size
}
throw BuildFSSync.Error.couldNotDetermineFileSize(self.cleanPath)
}
fileprivate func modTime() throws -> String {
let attrs = try FileManager.default.attributesOfItem(atPath: self.cleanPath)
if let date = attrs[FileAttributeKey.modificationDate] as? Date {
return date.rfc3339()
}
throw BuildFSSync.Error.couldNotDetermineModTime(self.cleanPath)
}
fileprivate func mode() throws -> UInt32 {
let attrs = try FileManager.default.attributesOfItem(atPath: self.cleanPath)
if let mode = attrs[FileAttributeKey.posixPermissions] as? NSNumber {
return mode.uint32Value
}
throw BuildFSSync.Error.couldNotDetermineFileMode(self.cleanPath)
}
}
extension Date {
fileprivate func rfc3339() -> String {
let dateFormatter = DateFormatter()
dateFormatter.dateFormat = "yyyy-MM-dd'T'HH:mm:ssZZZZZ"
dateFormatter.locale = Locale(identifier: "en_US_POSIX")
dateFormatter.timeZone = TimeZone(secondsFromGMT: 0) // Adjust if necessary
return dateFormatter.string(from: self)
}
}
extension String {
var cleanPathComponent: String {
let trimmed = self.trimmingCharacters(in: CharacterSet(charactersIn: "/"))
if let clean = trimmed.removingPercentEncoding {
return clean
}
return trimmed
}
}