From 420b915d8b4b0bc5d7edc638b985ee8fd32a3fbe Mon Sep 17 00:00:00 2001 From: Aditya Ramani Date: Wed, 18 Mar 2026 13:06:28 -0700 Subject: [PATCH] Write archive contents in chunks (#589) Fix a bug in ArchiveWriter where large files would only partly be written into an archive. This is done so by writing the data in chunks. --------- Signed-off-by: Aditya Ramani --- .../ArchiveError.swift | 6 +++ .../ArchiveReader.swift | 4 +- .../ArchiveWriter.swift | 43 ++++++++++++++++--- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/Sources/ContainerizationArchive/ArchiveError.swift b/Sources/ContainerizationArchive/ArchiveError.swift index 4e307738..0caee8b3 100644 --- a/Sources/ContainerizationArchive/ArchiveError.swift +++ b/Sources/ContainerizationArchive/ArchiveError.swift @@ -36,6 +36,8 @@ public enum ArchiveError: Error, CustomStringConvertible { case failedToDetectFilter case failedToDetectFormat case failedToExtractArchive(String) + case failedToCreateArchive(String) + case invalidBaseAddressArchiveWrite /// Description of the error public var description: String { @@ -74,6 +76,10 @@ public enum ArchiveError: Error, CustomStringConvertible { return "failed to detect format from archive." case .failedToExtractArchive(let reason): return "failed to extract archive: \(reason)" + case .failedToCreateArchive(let reason): + return "failed to create archive: \(reason)" + case .invalidBaseAddressArchiveWrite: + return "got an invalid base address for pointer when writing data to archive" } } } diff --git a/Sources/ContainerizationArchive/ArchiveReader.swift b/Sources/ContainerizationArchive/ArchiveReader.swift index b353b209..ab481f77 100644 --- a/Sources/ContainerizationArchive/ArchiveReader.swift +++ b/Sources/ContainerizationArchive/ArchiveReader.swift @@ -48,7 +48,7 @@ public struct ArchiveEntryReader: ReadableStream { /// A class responsible for reading entries from an archive file. public final class ArchiveReader { - private static let blockSize = 65536 + private static let chunkSize = 4 * 1024 * 1024 /// A pointer to the underlying `archive` C structure. var underlying: OpaquePointer? @@ -375,7 +375,7 @@ extension ArchiveReader { } private static func copyDataReaderToFd(dataReader: ArchiveEntryReader, fileFd: Int32, memberPath: FilePath) throws { - var buffer = [UInt8](repeating: 0, count: ArchiveReader.blockSize) + var buffer = [UInt8](repeating: 0, count: ArchiveReader.chunkSize) while true { let bytesRead = buffer.withUnsafeMutableBufferPointer { bufferPtr in guard let baseAddress = bufferPtr.baseAddress else { return 0 } diff --git a/Sources/ContainerizationArchive/ArchiveWriter.swift b/Sources/ContainerizationArchive/ArchiveWriter.swift index 8f35aa70..7d2d0792 100644 --- a/Sources/ContainerizationArchive/ArchiveWriter.swift +++ b/Sources/ContainerizationArchive/ArchiveWriter.swift @@ -20,6 +20,8 @@ import SystemPackage /// A class responsible for writing archives in various formats. public final class ArchiveWriter { + private static let chunkSize = 4 * 1024 * 1024 + var underlying: OpaquePointer! /// Initialize a new `ArchiveWriter` with the given configuration. @@ -166,9 +168,16 @@ extension ArchiveWriter { fileprivate func writeData(data: UnsafeRawBufferPointer) throws { guard let underlying = self.underlying else { throw ArchiveError.noUnderlyingArchive } - let result = archive_write_data(underlying, data.baseAddress, data.count) - guard result >= 0 else { - throw ArchiveError.unableToWriteData(result) + var offset = 0 + while offset < data.count { + guard let baseAddress = data.baseAddress?.advanced(by: offset) else { + throw ArchiveError.invalidBaseAddressArchiveWrite + } + let result = archive_write_data(underlying, baseAddress, data.count - offset) + guard result > 0 else { + throw ArchiveError.unableToWriteData(result) + } + offset += Int(result) } } } @@ -188,7 +197,7 @@ extension ArchiveWriter { var rootStat = stat() guard lstat(dirPath.string, &rootStat) == 0 else { let err = POSIXErrorCode(rawValue: errno) ?? .EINVAL - throw ArchiveError.failedToExtractArchive("lstat failed for '\(dirPath)': \(POSIXError(err))") + throw ArchiveError.failedToCreateArchive("lstat failed for '\(dirPath)': \(POSIXError(err))") } let rootEntry = WriteEntry() rootEntry.path = "./" @@ -215,7 +224,7 @@ extension ArchiveWriter { guard lstat(fullPath.string, &statInfo) == 0 else { let errNo = errno let err = POSIXErrorCode(rawValue: errNo) ?? .EINVAL - throw ArchiveError.failedToExtractArchive("lstat failed for '\(fullPath)': \(POSIXError(err))") + throw ArchiveError.failedToCreateArchive("lstat failed for '\(fullPath)': \(POSIXError(err))") } let mode = statInfo.st_mode @@ -267,8 +276,28 @@ extension ArchiveWriter { entry.owner = uid entry.permissions = mode if type == .regular { - let data = try Data(contentsOf: URL(fileURLWithPath: fullPath.string), options: .uncached) - try self.writeEntry(entry: entry, data: data) + let buf = UnsafeMutableRawBufferPointer.allocate(byteCount: Self.chunkSize, alignment: 1) + guard let baseAddress = buf.baseAddress else { + throw ArchiveError.failedToCreateArchive("cannot create temporary buffer of size \(Self.chunkSize)") + } + defer { buf.deallocate() } + let fd = Foundation.open(fullPath.string, O_RDONLY) + guard fd >= 0 else { + let err = POSIXErrorCode(rawValue: errno) ?? .EINVAL + throw ArchiveError.failedToCreateArchive("cannot open file \(fullPath.string) for reading: \(err)") + } + defer { close(fd) } + try self.writeHeader(entry: entry) + while true { + let n = read(fd, baseAddress, Self.chunkSize) + if n == 0 { break } + if n < 0 { + let err = POSIXErrorCode(rawValue: errno) ?? .EIO + throw ArchiveError.failedToCreateArchive("failed to read from file \(fullPath.string): \(err)") + } + try self.writeData(data: UnsafeRawBufferPointer(start: baseAddress, count: n)) + } + try self.finishEntry() } else { try self.writeEntry(entry: entry, data: nil) }