feat: virtiofs rootfs hotplug for LinuxPod on cloud-hypervisor (#809)

Allow a container hotplugged into an already-running LinuxPod VM
(addContainer after create()) to have a virtiofs (directory-share)
rootfs, not just an ext4 virtio-blk image, on the cloud-hypervisor
backend.

Host side:
- Generalize CHHotplugProvider.hotplug to branch on the rootfs
runtimeOptions (.virtioblk / .virtiofs / unsupported), reusing the
per-tag virtiofsd/vm.add-fs machinery factored into a new
ensureVirtiofsDevice helper. Make registerMounts the sole writer of the
per-VM mount registry so the rootfs is always first and mounts are not
double-registered (also fixes a latent ordering bug on the block path).
Exclude the rootfs from the /run/virtiofs holding-dir loop in
addContainer.

Guest side (fixes a pre-existing, feature-independent bug):
- The addContainer-after-create (runtime hotplug) path had never been
exercised by any test. cloud-hypervisor adds hot-plugged
virtio-blk/virtio-fs devices to the guest PCI bus but the guest does not
auto-probe them, so a mount by /dev/vdX or virtio-fs tag fails (ENOENT /
tag-not-found) for ALL CH hotplug.
- vminitd's mount RPC now, on a mount failure for a hot-plug-candidate
source (virtiofs type or /dev/vd* source), forces /sys/bus/pci/rescan
and retries with a bounded wait. Scoped so boot-time and non-device
mounts are unaffected.

Tests / build:
- First coverage of LinuxPod.addContainer after create() (CH-only; VZ
has no runtime hotplug): pod hotplug virtiofs rootfs and pod hotplug
block rootfs, registered in a Linux-only test list, plus a
directory-rootfs unpack helper.
- make linux-integration gains an optional FILTER passthrough (make
linux-integration FILTER="pod hotplug").

Signed-off-by: michael_crosby <michael_crosby@apple.com>

Makefile

@@ -35,6 +35,12 @@ LINUX_INTEGRATION_KERNEL := kernel/vmlinuz-x86_64
 else
 LINUX_INTEGRATION_KERNEL := kernel/vmlinux-$(KERNEL_ARCH)
 endif
+
+# Optional test-name filter for `make linux-integration`, e.g.
+#   make linux-integration FILTER="pod hotplug"
+# Comma-separated; a test is kept if its name contains ANY of the substrings.
+FILTER ?=
+linux_integration_filter = $(if $(strip $(FILTER)),--filter '$(strip $(FILTER))')
 ifeq ($(UNAME_S),Darwin)
 SWIFT ?= /usr/bin/swift
 else
@@ -239,7 +245,7 @@ ifeq (,$(wildcard bin/initfs.ext4))
 	@echo "missing bin/initfs.ext4; run 'make init' first (this also seeds the persistent imageStore at .local/integration-cache)"
 	@exit 1
 endif
-	$(call linux_run,CONTAINERIZATION_RELAXED_SANDBOX=1 ./bin/containerization-integration --kernel ./$(LINUX_INTEGRATION_KERNEL) --ch-binary ./bin/cloud-hypervisor --virtiofsd-binary ./bin/virtiofsd --max-concurrency 1,--kernel $(LINUX_INTEGRATION_KERNEL))
+	$(call linux_run,CONTAINERIZATION_RELAXED_SANDBOX=1 ./bin/containerization-integration --kernel ./$(LINUX_INTEGRATION_KERNEL) --ch-binary ./bin/cloud-hypervisor --virtiofsd-binary ./bin/virtiofsd --max-concurrency 1 $(linux_integration_filter),--kernel $(LINUX_INTEGRATION_KERNEL))
 
 # Builds the x86_64 deployment tarball.
 #

Sources/Containerization/CHHotplugProvider.swift

@@ -93,39 +93,61 @@ final class CHHotplugProvider: HotplugProvider {
 
     // MARK: - HotplugProvider conformance
 
-    func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem {
-        guard case .virtioblk = block.runtimeOptions else {
-            throw ContainerizationError(.invalidArgument, message: "hotplug requires a virtio-blk mount")
+    func hotplug(_ rootfs: Mount, id: String) async throws -> AttachedFilesystem {
+        switch rootfs.runtimeOptions {
+        case .virtioblk:
+            let letter = try allocator.allocate()
+            let chId = "blk-\(id)-\(letter)"
+            let disk = CloudHypervisor.DiskConfig(
+                path: rootfs.source,
+                readonly: rootfs.options.contains("ro"),
+                id: chId,
+                imageType: .raw
+            )
+
+            let pci: CloudHypervisor.PciDeviceInfo
+            do {
+                pci = try await chCall { try await self.client.vmAddDisk(disk) }
+            } catch {
+                try? allocator.release(letter)
+                throw error
+            }
+
+            let attached = AttachedFilesystem(
+                type: rootfs.type,
+                source: "/dev/vd\(letter)",
+                destination: rootfs.destination,
+                options: rootfs.options
+            )
+
+            _records.withLock {
+                $0[id, default: []].append(HotplugRecord(chDeviceId: pci.id, kind: .block(letter: letter)))
+            }
+            return attached
+
+        case .virtiofs:
+            // Compute the tag up front (throwing) so nothing can fail between
+            // committing the virtiofsd/device and recording the HotplugRecord —
+            // otherwise a thrown error would orphan a running virtiofsd.
+            let tag = try hashFilePath(path: rootfs.source)
+            let chDeviceId = try await ensureVirtiofsDevice(
+                tag: tag,
+                source: rootfs.source,
+                readonly: rootfs.options.contains("ro")
+            )
+            _records.withLock {
+                $0[id, default: []].append(HotplugRecord(chDeviceId: chDeviceId, kind: .virtiofs(tag: tag)))
+            }
+            return AttachedFilesystem(
+                type: rootfs.type,
+                source: tag,
+                destination: rootfs.destination,
+                options: rootfs.options
+            )
+
+        case .shared, .any:
+            throw ContainerizationError(.unsupported, message: "hotplug rootfs must be virtio-blk or virtiofs")
         }
-
-        let letter = try allocator.allocate()
-        let chId = "blk-\(id)-\(letter)"
-        let disk = CloudHypervisor.DiskConfig(
-            path: block.source,
-            readonly: block.options.contains("ro"),
-            id: chId,
-            imageType: .raw
-        )
-
-        let pci: CloudHypervisor.PciDeviceInfo
-        do {
-            pci = try await chCall { try await self.client.vmAddDisk(disk) }
-        } catch {
-            try? allocator.release(letter)
-            throw error
-        }
-
-        let attached = AttachedFilesystem(
-            type: block.type,
-            source: "/dev/vd\(letter)",
-            destination: block.destination,
-            options: block.options
-        )
-
-        _records.withLock {
-            $0[id, default: []].append(HotplugRecord(chDeviceId: pci.id, kind: .block(letter: letter)))
-        }
-        return attached
     }
 
     func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws {
@@ -199,96 +221,78 @@ final class CHHotplugProvider: HotplugProvider {
         }
 
         for (tag, group) in byTag {
-            // Hold spawnLock across the existence check and the spawn /
-            // _tags write so two concurrent calls for the same tag can't
-            // both decide alreadyRunning=false and double-spawn virtiofsd
-            // (the second write would clobber the first in `_tags`,
-            // orphaning that process).
-            try await spawnLock.withLock { _ in
-                // Build per-container AttachedFilesystem entries up front.
-                // These depend only on Mount + allocator and don't need the
-                // chDeviceId, so surfacing any error here keeps the
-                // transactional shape: nothing irreversible has happened
-                // yet, no virtiofsd has spawned, no _tags entry written.
-                var attached: [AttachedFilesystem] = []
-                for mount in group {
-                    attached.append(try AttachedFilesystem(mount: mount, allocator: self.allocator))
-                }
-
-                let chDeviceId: String
-
-                // Refcount-bump path. If a virtiofsd already serves this
-                // tag, increment refcount and use the cached deviceId.
-                let cachedDeviceId: String? = self._tags.withLock { tags in
-                    if var state = tags[tag] {
-                        state.refcount += 1
-                        tags[tag] = state
-                        return state.chDeviceId
-                    }
-                    return nil
-                }
-
-                if let cached = cachedDeviceId {
-                    chDeviceId = cached
-                } else {
-                    // First-spawn path. Walk: spawn → vmAddFs → commit _tags,
-                    // with rollback at every step so a partial failure can't
-                    // leave a virtiofsd running unrecorded.
-                    let socket = chVirtiofsSocketURL(workDir: self.workDir, tag: tag)
-                    let readonly = group.allSatisfy { $0.options.contains("ro") }
-                    guard let source = group.first?.source else { return }
-                    let virtiofsdBinary = try CHVirtualMachineManager.resolveBinary(
-                        self.virtiofsdBinaryOverride,
-                        name: "virtiofsd"
-                    )
-
-                    let process = VirtiofsdProcess(
-                        config: .init(
-                            binary: virtiofsdBinary,
-                            socketPath: socket,
-                            sharedDir: URL(fileURLWithPath: source),
-                            readonly: readonly
-                        ),
-                        logger: self.logger
-                    )
-
-                    try await process.start()
-
-                    let fsConfig = CloudHypervisor.FsConfig(
-                        tag: tag,
-                        socket: socket.path,
-                        id: "fs-\(tag)"
-                    )
-                    let pci: CloudHypervisor.PciDeviceInfo
-                    do {
-                        pci = try await chCall { try await self.client.vmAddFs(fsConfig) }
-                    } catch {
-                        await process.terminate(graceSeconds: 5)
-                        try? FileManager.default.removeItem(at: socket)
-                        throw error
-                    }
-
-                    self._tags.withLock {
-                        $0[tag] = VirtiofsdTagState(process: process, refcount: 1, chDeviceId: pci.id)
-                    }
-                    chDeviceId = pci.id
-                }
-
-                // Bookkeeping. Both writes are non-throwing closures, and
-                // `attached` was built up front, so once we reach here
-                // nothing can fail between the refcount/spawn commit above
-                // and the per-container record below — the orphan window
-                // (tag committed, record missing) is closed.
-                self._records.withLock {
-                    $0[id, default: []].append(HotplugRecord(chDeviceId: chDeviceId, kind: .virtiofs(tag: tag)))
-                }
-                self._mounts.withLock {
-                    $0[id, default: []].append(contentsOf: attached)
-                }
+            guard let source = group.first?.source else { continue }
+            let readonly = group.allSatisfy { $0.options.contains("ro") }
+            let chDeviceId = try await ensureVirtiofsDevice(tag: tag, source: source, readonly: readonly)
+            // Record once per tag for this container. The AttachedFilesystem
+            // entries for these mounts are written by registerMounts (the sole
+            // _mounts writer), so we do NOT touch _mounts here.
+            _records.withLock {
+                $0[id, default: []].append(HotplugRecord(chDeviceId: chDeviceId, kind: .virtiofs(tag: tag)))
             }
         }
     }
 
+    /// Ensure a virtio-fs device backed by `virtiofsd` exists for `tag`,
+    /// spawning one (and issuing `vm.add-fs`) on first use or bumping the
+    /// refcount of an existing one. Returns the cloud-hypervisor device id
+    /// (`vm.remove-device` keys on it). Serialized per-provider by `spawnLock`
+    /// so two concurrent callers for the same tag can't double-spawn.
+    private func ensureVirtiofsDevice(tag: String, source: String, readonly: Bool) async throws -> String {
+        try await spawnLock.withLock { _ in
+            // Refcount-bump path: a virtiofsd already serves this tag.
+            let cached: String? = self._tags.withLock { tags in
+                if var state = tags[tag] {
+                    state.refcount += 1
+                    tags[tag] = state
+                    return state.chDeviceId
+                }
+                return nil
+            }
+            if let cached {
+                return cached
+            }
+
+            // First-spawn path: spawn → vm.add-fs → commit _tags, rolling back
+            // the process/socket if vm.add-fs fails.
+            let socket = chVirtiofsSocketURL(workDir: self.workDir, tag: tag)
+            let virtiofsdBinary = try CHVirtualMachineManager.resolveBinary(
+                self.virtiofsdBinaryOverride,
+                name: "virtiofsd"
+            )
+            let process = VirtiofsdProcess(
+                config: .init(
+                    binary: virtiofsdBinary,
+                    socketPath: socket,
+                    sharedDir: URL(fileURLWithPath: source),
+                    readonly: readonly
+                ),
+                logger: self.logger
+            )
+
+            try await process.start()
+
+            let fsConfig = CloudHypervisor.FsConfig(
+                tag: tag,
+                socket: socket.path,
+                id: "fs-\(tag)"
+            )
+            let pci: CloudHypervisor.PciDeviceInfo
+            do {
+                pci = try await chCall { try await self.client.vmAddFs(fsConfig) }
+            } catch {
+                await process.terminate(graceSeconds: 5)
+                try? FileManager.default.removeItem(at: socket)
+                throw error
+            }
+
+            self._tags.withLock {
+                $0[tag] = VirtiofsdTagState(process: process, refcount: 1, chDeviceId: pci.id)
+            }
+            return pci.id
+        }
+    }
+
     func releaseVirtioFS(id: String) async throws {
         let popped: [HotplugRecord] = _records.withLock { records in
             let all = records[id] ?? []

Sources/Containerization/LinuxPod.swift

@@ -438,14 +438,18 @@ extension LinuxPod {
                             additionalMounts: nonSharedMounts
                         )
 
-                        // Mount the hotplugged container's virtiofs shares in the
+                        // Mount this container's additional virtiofs shares in the
                         // guest. create() does this for boot-time containers (the
                         // /run/virtiofs loop); the hotplug path must do the same or
                         // the container's bind mounts from /run/virtiofs/<tag> fail
                         // with ENOENT.
-                        let newVirtiofsTags = (vm.mounts[id] ?? [])
-                            .filter { $0.type == "virtiofs" }
-                            .map { $0.source }
+                        //
+                        // Derive the tags from the additional mounts directly rather
+                        // than from vm.mounts[id], so this is independent of the
+                        // rootfs (which may be virtiofs or virtio-blk) and of mount
+                        // ordering. The rootfs is mounted at /run/container/<id>/rootfs
+                        // and is never consumed from /run/virtiofs.
+                        let newVirtiofsTags = try virtioFSMounts.map { try hashFilePath(path: $0.source) }
                         if !newVirtiofsTags.isEmpty {
                             // Tags already mounted in the guest at boot or by a
                             // prior hotplug (i.e. present on another container).

Sources/Integration/PodTests.swift

@@ -16,6 +16,7 @@
 
 import ArgumentParser
 import Containerization
+import ContainerizationArchive
 import ContainerizationEXT4
 import ContainerizationError
 import ContainerizationExtras
@@ -2221,4 +2222,145 @@ extension IntegrationSuite {
             throw error
         }
     }
+
+    #if os(Linux)
+    /// Unpack an image's layers into a host directory to use as a virtiofs
+    /// (directory-share) rootfs. Assumes a single-layer image (the alpine
+    /// image used by the suite) so no OCI whiteout processing is required.
+    ///
+    /// The extracted dir lives under `Self.testDir`; do NOT `defer`-remove it
+    /// here — virtiofsd shares it for the whole test. It is swept by
+    /// `bootstrap`'s `maxConcurrency == 1` reaper on the next test and by the
+    /// suite-end `removeItem(at: Self.testDir)`.
+    private func unpackRootfsDirectory(_ image: Containerization.Image, testID: String) async throws -> Containerization.Mount {
+        let dir = Self.testDir.appending(component: "\(testID)-rootfs-dir")
+        try? FileManager.default.removeItem(at: dir)
+        try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
+
+        let platform = Platform(arch: "arm64", os: "linux", variant: "v8")
+        let manifest = try await image.manifest(for: platform)
+        for layer in manifest.layers {
+            let content = try await image.getContent(digest: layer.digest)
+            let filter: ContainerizationArchive.Filter
+            switch layer.mediaType {
+            case MediaTypes.imageLayer, MediaTypes.dockerImageLayer:
+                filter = .none
+            case MediaTypes.imageLayerGzip, MediaTypes.dockerImageLayerGzip:
+                filter = .gzip
+            case MediaTypes.imageLayerZstd, MediaTypes.dockerImageLayerZstd:
+                filter = .zstd
+            default:
+                throw IntegrationError.assert(msg: "unsupported layer media type \(layer.mediaType)")
+            }
+            let reader = try ArchiveReader(format: .paxRestricted, filter: filter, file: content.path)
+            _ = try reader.extractContents(to: dir)
+        }
+
+        return .share(source: dir.absolutePath(), destination: "/")
+    }
+
+    /// Hotplug a container with a virtiofs (directory-share) rootfs into a
+    /// running pod VM, plus an additional virtiofs file-mount. CH-only: VZ has
+    /// no runtime hotplug.
+    func testPodHotplugVirtiofsRootfs() async throws {
+        let id = "test-pod-hotplug-virtiofs-rootfs"
+        let bs = try await bootstrap(id)
+
+        let pod = try LinuxPod(id, vmm: bs.vmm) { config in
+            config.cpus = 4
+            config.memoryInBytes = 1024.mib()
+            config.bootLog = bs.bootLog
+        }
+
+        // Boot-time seed container (block rootfs) so the target container is
+        // added strictly on the post-create (hotplug) path.
+        try await pod.addContainer("seed", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "seed")) { config in
+            config.process.arguments = ["/bin/sleep", "infinity"]
+        }
+
+        try await pod.create()
+
+        // Also hotplug a virtiofs file-mount onto the container. This exercises
+        // the /run/virtiofs holding-dir path for an additional virtiofs share on
+        // a container whose rootfs is itself virtiofs — the combination the
+        // rootfs-agnostic newVirtiofsTags derivation must handle. cat'ing the
+        // mounted file also confirms the virtiofs rootfs itself is usable.
+        let mountContent = "hello from hotplugged virtiofs file mount"
+        let hostFile = FileManager.default.uniqueTemporaryDirectory(create: true)
+            .appendingPathComponent("hotplug-mount.txt")
+        try mountContent.write(to: hostFile, atomically: true, encoding: .utf8)
+
+        let virtiofsRootfs = try await unpackRootfsDirectory(bs.image, testID: id)
+        let buffer = BufferWriter()
+        try await pod.addContainer("hot", rootfs: virtiofsRootfs) { config in
+            config.process.arguments = ["/bin/cat", "/etc/hotplug-mount.txt"]
+            config.mounts.append(.share(source: hostFile.path, destination: "/etc/hotplug-mount.txt"))
+            config.process.stdout = buffer
+        }
+
+        do {
+            try await pod.startContainer("hot")
+            let status = try await pod.waitContainer("hot")
+
+            try await pod.stopContainer("hot")
+            try await pod.stop()
+
+            guard status.exitCode == 0 else {
+                throw IntegrationError.assert(msg: "hot container status \(status) != 0")
+            }
+            guard String(data: buffer.data, encoding: .utf8) == mountContent else {
+                throw IntegrationError.assert(
+                    msg: "expected '\(mountContent)', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'")
+            }
+        } catch {
+            try? await pod.stop()
+            throw error
+        }
+    }
+
+    /// Hotplug a container with a block rootfs into a running pod VM. Guards
+    /// the existing block hotplug path against the registry-consolidation
+    /// change. CH-only.
+    func testPodHotplugBlockRootfs() async throws {
+        let id = "test-pod-hotplug-block-rootfs"
+        let bs = try await bootstrap(id)
+
+        let pod = try LinuxPod(id, vmm: bs.vmm) { config in
+            config.cpus = 4
+            config.memoryInBytes = 1024.mib()
+            config.bootLog = bs.bootLog
+        }
+
+        try await pod.addContainer("seed", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "seed")) { config in
+            config.process.arguments = ["/bin/sleep", "infinity"]
+        }
+
+        try await pod.create()
+
+        let buffer = BufferWriter()
+        try await pod.addContainer("hot", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "hot")) { config in
+            config.process.arguments = ["/bin/echo", "hello from block rootfs"]
+            config.process.stdout = buffer
+        }
+
+        do {
+            try await pod.startContainer("hot")
+            let status = try await pod.waitContainer("hot")
+
+            try await pod.stopContainer("hot")
+            try await pod.stop()
+
+            guard status.exitCode == 0 else {
+                throw IntegrationError.assert(msg: "hot container status \(status) != 0")
+            }
+            guard String(data: buffer.data, encoding: .utf8) == "hello from block rootfs\n" else {
+                throw IntegrationError.assert(
+                    msg: "expected 'hello from block rootfs', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'")
+            }
+        } catch {
+            try? await pod.stop()
+            throw error
+        }
+    }
+    #endif
 }

Sources/Integration/Suite.swift

@@ -621,7 +621,13 @@ struct IntegrationSuite: AsyncParsableCommand {
             ] + macOS26Tests()
         let tests: [Test] = crossPlatformTests + macOSOnlyTests
         #else
-        let tests: [Test] = crossPlatformTests
+        // Hotplug into a running pod VM is CH-only (VZ has no runtime hotplug),
+        // and no pod test elsewhere exercises addContainer-after-create.
+        let linuxOnlyTests: [Test] = [
+            Test("pod hotplug block rootfs", testPodHotplugBlockRootfs),
+            Test("pod hotplug virtiofs rootfs", testPodHotplugVirtiofsRootfs),
+        ]
+        let tests: [Test] = crossPlatformTests + linuxOnlyTests
         #endif
 
         let filteredTests: [Test]

vminitd/Sources/VminitdCore/Server+GRPC.swift

@@ -663,7 +663,47 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ
             )
 
             #if os(Linux)
-            try mnt.mount(createWithPerms: 0o755)
+            do {
+                try mnt.mount(createWithPerms: 0o755)
+            } catch {
+                // A hot-plugged virtio device (virtio-blk / virtio-fs) may not
+                // be enumerated by the guest yet when the host issues this
+                // mount immediately after vm.add-disk / vm.add-fs: cloud-
+                // hypervisor places the device on the PCI bus but the guest
+                // does not always auto-probe it. Force a PCI rescan and retry
+                // with a bounded wait. Scoped to hot-plug-candidate sources so
+                // an ordinary mount failure isn't delayed.
+                let hotplugCandidate = request.type == "virtiofs" || request.source.hasPrefix("/dev/vd")
+                guard hotplugCandidate else { throw error }
+
+                if let rescan = FileHandle(forWritingAtPath: "/sys/bus/pci/rescan") {
+                    defer { try? rescan.close() }
+                    do {
+                        try rescan.write(contentsOf: Data("1".utf8))
+                        log.info("mount: triggered PCI bus rescan for hot-plugged device")
+                    } catch {
+                        log.error("mount: PCI rescan write failed", metadata: ["error": "\(error)"])
+                    }
+                } else {
+                    log.error("mount: cannot open /sys/bus/pci/rescan")
+                }
+
+                var mounted = false
+                for attempt in 1...20 {  // up to ~2s for the device to enumerate
+                    try? await Task.sleep(for: .milliseconds(100))
+                    do {
+                        try mnt.mount(createWithPerms: 0o755)
+                        log.info("mount: succeeded after PCI rescan", metadata: ["attempt": "\(attempt)"])
+                        mounted = true
+                        break
+                    } catch {
+                        continue
+                    }
+                }
+                if !mounted {
+                    throw error
+                }
+            }
             return .init()
             #else
             fatalError("mount not supported on platform")