lee/container
1
0
Fork 0
mirror of https://github.com/apple/container.git synced 2026-09-12 10:45:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fetch without chunking in parallel (#75)

Browse Source 
## Issue

When pulling images, the download speed appears to be slower compared to
Docker.
I found that parallel chunk generation was being performed during the
layer download process.
While individual chunks allow parallel download operations, the overall
process remains sequential between chunks.
This results less performance when chunks contain both small and large
layers mixed together.

## Changes

Discontinued chunk-based segmentation to enable more efficient parallel
downloads.

## Results(in my local env)

| image | layers | old | new |
| ---|---|---|---|
| node:latest | 12 | 1m15s | 1m10s |
| ghcr.io/norio-nomura/swift_discord_bot:main | 54 |  2m45s | 2m30s |

<details>
<summary>raw terminal log</summary>

```
❯ rm -rf ~/Library/Application\ Support/com.apple.containerization && time bin/cctl_old images pull docker.io/library/node:latest
image pulled
bin/cctl_old images pull docker.io/library/node:latest  42.37s user 6.60s system 64% cpu 1:15.94 total

❯ rm -rf ~/Library/Application\ Support/com.apple.containerization && time bin/cctl_new images pull docker.io/library/node:latest
image pulled
bin/cctl_new images pull docker.io/library/node:latest  42.27s user 6.72s system 69% cpu 1:10.58 total

❯ rm -rf ~/Library/Application\ Support/com.apple.containerization && time bin/cctl_old images pull docker.io/library/node:latest
image pulled
bin/cctl_old images pull docker.io/library/node:latest  45.65s user 7.36s system 70% cpu 1:15.64 total

❯ rm -rf ~/Library/Application\ Support/com.apple.containerization && time bin/cctl_new images pull docker.io/library/node:latest
image pulled
bin/cctl_new images pull docker.io/library/node:latest  39.76s user 6.32s system 65% cpu 1:10.50 total

❯ rm -rf ~/Library/Application\ Support/com.apple.containerization && time bin/cctl_old images pull docker.io/library/node:latest
image pulled
bin/cctl_old images pull docker.io/library/node:latest  42.47s user 6.75s system 65% cpu 1:14.72 total

❯ rm -rf ~/Library/Application\ Support/com.apple.containerization && time bin/cctl_new images pull docker.io/library/node:latest
image pulled
bin/cctl_new images pull docker.io/library/node:latest  42.28s user 6.65s system 69% cpu 1:09.93 total

❯ rm -rf ~/Library/Application\ Support/com.apple.containerization && time bin/cctl_new images pull ghcr.io/norio-nomura/swift_discord_bot:main
image pulled
bin/cctl_new images pull ghcr.io/norio-nomura/swift_discord_bot:main  103.83s user 18.71s system 81% cpu 2:30.02 total

❯ rm -rf ~/Library/Application\ Support/com.apple.containerization && time bin/cctl_old images pull ghcr.io/norio-nomura/swift_discord_bot:main
image pulled
bin/cctl_old images pull ghcr.io/norio-nomura/swift_discord_bot:main  120.79s user 20.70s system 85% cpu 2:45.26 total
```
</details>


<details>
<summary>patch for download only</summary>

```diff
diff --git a/Sources/cctl/ImageCommand.swift b/Sources/cctl/ImageCommand.swift
index 84c5218..4aa4bb8 100644
--- a/Sources/cctl/ImageCommand.swift
+++ b/Sources/cctl/ImageCommand.swift
@@ -127,6 +127,7 @@ extension Application {
                 }
 
                 print("image pulled")
+                return
 
                 let tempDir = FileManager.default.uniqueTemporaryDirectory(create: true)
                 if let platform {
```
</details>
This commit is contained in:
Iceman
2025-06-10 21:26:36 -07:00
committed by GitHub
parent 4ad3c996f0
commit 5258424e36
1 changed files with 36 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Sources/Containerization/Image/ImageStore/ImageStore+Import.swift
+36 -24
View File
@@ -56,7 +56,7 @@ extension ImageStore {
])
}
try await self.fetch(toProcess)
try await self.fetchAll(toProcess)
let children = try await self.walk(toProcess)
let filtered = try filterPlatforms(matcher: matcher, children)
toProcess = filtered.uniqued { $0.digest }
@@ -118,37 +118,49 @@ extension ImageStore {
return out
}
private func fetch(_ inDesc: [Descriptor]) async throws {
private func fetchAll(_ descriptors: [Descriptor]) async throws {
try await withThrowingTaskGroup(of: Void.self) { group in
for chunk in inDesc.chunks(ofCount: 8) {
for desc in chunk {
if let found = try await self.contentStore.get(digest: desc.digest) {
try FileManager.default.copyItem(at: found.path, to: ingestDir.appendingPathComponent(desc.digest.trimmingDigestPrefix))
await progress?([
// Count the size of the blob
ProgressEvent(event: "add-size", value: desc.size),
// Count the number of blobs
ProgressEvent(event: "add-items", value: 1),
])
continue
}
var iterator = descriptors.makeIterator()
for _ in 0..<8 {
if let desc = iterator.next() {
group.addTask {
if desc.size > 1.mib() {
try await self.fetchBlob(desc)
} else {
try await self.fetchData(desc)
}
// Count the number of blobs
await progress?([
ProgressEvent(event: "add-items", value: 1)
])
try await fetch(desc)
}
}
}
for try await _ in group {
if let desc = iterator.next() {
group.addTask {
try await fetch(desc)
}
}
try await group.waitForAll()
}
}
}
private func fetch(_ descriptor: Descriptor) async throws {
if let found = try await self.contentStore.get(digest: descriptor.digest) {
try FileManager.default.copyItem(at: found.path, to: ingestDir.appendingPathComponent(descriptor.digest.trimmingDigestPrefix))
await progress?([
// Count the size of the blob
ProgressEvent(event: "add-size", value: descriptor.size),
// Count the number of blobs
ProgressEvent(event: "add-items", value: 1),
])
return
}
if descriptor.size > 1.mib() {
try await self.fetchBlob(descriptor)
} else {
try await self.fetchData(descriptor)
}
// Count the number of blobs
await progress?([
ProgressEvent(event: "add-items", value: 1)
])
}
private func fetchBlob(_ descriptor: Descriptor) async throws {
let id = UUID().uuidString
let fm = FileManager.default