From 5258424e3622ceb7f8cc03e4db90f7783e7cb929 Mon Sep 17 00:00:00 2001 From: Iceman Date: Wed, 11 Jun 2025 13:26:36 +0900 Subject: [PATCH] Fetch without chunking in parallel (#75) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Issue When pulling images, the download speed appears to be slower compared to Docker. I found that parallel chunk generation was being performed during the layer download process. While individual chunks allow parallel download operations, the overall process remains sequential between chunks. This results less performance when chunks contain both small and large layers mixed together. ## Changes Discontinued chunk-based segmentation to enable more efficient parallel downloads. ## Results(in my local env) | image | layers | old | new | | ---|---|---|---| | node:latest | 12 | 1m15s | 1m10s | | ghcr.io/norio-nomura/swift_discord_bot:main | 54 | 2m45s | 2m30s |
raw terminal log ``` ❯ rm -rf ~/Library/Application\ Support/com.apple.containerization && time bin/cctl_old images pull docker.io/library/node:latest image pulled bin/cctl_old images pull docker.io/library/node:latest 42.37s user 6.60s system 64% cpu 1:15.94 total ❯ rm -rf ~/Library/Application\ Support/com.apple.containerization && time bin/cctl_new images pull docker.io/library/node:latest image pulled bin/cctl_new images pull docker.io/library/node:latest 42.27s user 6.72s system 69% cpu 1:10.58 total ❯ rm -rf ~/Library/Application\ Support/com.apple.containerization && time bin/cctl_old images pull docker.io/library/node:latest image pulled bin/cctl_old images pull docker.io/library/node:latest 45.65s user 7.36s system 70% cpu 1:15.64 total ❯ rm -rf ~/Library/Application\ Support/com.apple.containerization && time bin/cctl_new images pull docker.io/library/node:latest image pulled bin/cctl_new images pull docker.io/library/node:latest 39.76s user 6.32s system 65% cpu 1:10.50 total ❯ rm -rf ~/Library/Application\ Support/com.apple.containerization && time bin/cctl_old images pull docker.io/library/node:latest image pulled bin/cctl_old images pull docker.io/library/node:latest 42.47s user 6.75s system 65% cpu 1:14.72 total ❯ rm -rf ~/Library/Application\ Support/com.apple.containerization && time bin/cctl_new images pull docker.io/library/node:latest image pulled bin/cctl_new images pull docker.io/library/node:latest 42.28s user 6.65s system 69% cpu 1:09.93 total ❯ rm -rf ~/Library/Application\ Support/com.apple.containerization && time bin/cctl_new images pull ghcr.io/norio-nomura/swift_discord_bot:main image pulled bin/cctl_new images pull ghcr.io/norio-nomura/swift_discord_bot:main 103.83s user 18.71s system 81% cpu 2:30.02 total ❯ rm -rf ~/Library/Application\ Support/com.apple.containerization && time bin/cctl_old images pull ghcr.io/norio-nomura/swift_discord_bot:main image pulled bin/cctl_old images pull ghcr.io/norio-nomura/swift_discord_bot:main 120.79s user 20.70s system 85% cpu 2:45.26 total ```
patch for download only ```diff diff --git a/Sources/cctl/ImageCommand.swift b/Sources/cctl/ImageCommand.swift index 84c5218..4aa4bb8 100644 --- a/Sources/cctl/ImageCommand.swift +++ b/Sources/cctl/ImageCommand.swift @@ -127,6 +127,7 @@ extension Application { } print("image pulled") + return let tempDir = FileManager.default.uniqueTemporaryDirectory(create: true) if let platform { ```
--- .../Image/ImageStore/ImageStore+Import.swift | 60 +++++++++++-------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/Sources/Containerization/Image/ImageStore/ImageStore+Import.swift b/Sources/Containerization/Image/ImageStore/ImageStore+Import.swift index da03a7e7..1b33d3b9 100644 --- a/Sources/Containerization/Image/ImageStore/ImageStore+Import.swift +++ b/Sources/Containerization/Image/ImageStore/ImageStore+Import.swift @@ -56,7 +56,7 @@ extension ImageStore { ]) } - try await self.fetch(toProcess) + try await self.fetchAll(toProcess) let children = try await self.walk(toProcess) let filtered = try filterPlatforms(matcher: matcher, children) toProcess = filtered.uniqued { $0.digest } @@ -118,37 +118,49 @@ extension ImageStore { return out } - private func fetch(_ inDesc: [Descriptor]) async throws { + private func fetchAll(_ descriptors: [Descriptor]) async throws { try await withThrowingTaskGroup(of: Void.self) { group in - for chunk in inDesc.chunks(ofCount: 8) { - for desc in chunk { - if let found = try await self.contentStore.get(digest: desc.digest) { - try FileManager.default.copyItem(at: found.path, to: ingestDir.appendingPathComponent(desc.digest.trimmingDigestPrefix)) - await progress?([ - // Count the size of the blob - ProgressEvent(event: "add-size", value: desc.size), - // Count the number of blobs - ProgressEvent(event: "add-items", value: 1), - ]) - continue - } + var iterator = descriptors.makeIterator() + for _ in 0..<8 { + if let desc = iterator.next() { group.addTask { - if desc.size > 1.mib() { - try await self.fetchBlob(desc) - } else { - try await self.fetchData(desc) - } - // Count the number of blobs - await progress?([ - ProgressEvent(event: "add-items", value: 1) - ]) + try await fetch(desc) + } + } + } + for try await _ in group { + if let desc = iterator.next() { + group.addTask { + try await fetch(desc) } } - try await group.waitForAll() } } } + private func fetch(_ descriptor: Descriptor) async throws { + if let found = try await self.contentStore.get(digest: descriptor.digest) { + try FileManager.default.copyItem(at: found.path, to: ingestDir.appendingPathComponent(descriptor.digest.trimmingDigestPrefix)) + await progress?([ + // Count the size of the blob + ProgressEvent(event: "add-size", value: descriptor.size), + // Count the number of blobs + ProgressEvent(event: "add-items", value: 1), + ]) + return + } + + if descriptor.size > 1.mib() { + try await self.fetchBlob(descriptor) + } else { + try await self.fetchData(descriptor) + } + // Count the number of blobs + await progress?([ + ProgressEvent(event: "add-items", value: 1) + ]) + } + private func fetchBlob(_ descriptor: Descriptor) async throws { let id = UUID().uuidString let fm = FileManager.default