feat: add --init-image flag for specifying custom init filesystem images per VM (#937)

- Users can now specify an alternate image to use
  with the `container run ---init-image` flag.

Makefile

@@ -191,6 +191,7 @@ integration: init-block
 		$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIStatsCommand || exit_code=1 ; \
 		$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIImagesCommand || exit_code=1 ; \
 		$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunBase || exit_code=1 ; \
+		$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunInitImage || exit_code=1 ; \
 		$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIBuildBase || exit_code=1 ; \
 		$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIVolumes || exit_code=1 ; \
 		$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIKernelSet || exit_code=1 ; \

Sources/ContainerCommands/Container/ContainerCreate.swift

@@ -84,7 +84,7 @@ extension Application {
 
             let options = ContainerCreateOptions(autoRemove: managementFlags.remove)
             let client = ContainerClient()
-            try await client.create(configuration: ck.0, options: options, kernel: ck.1)
+            try await client.create(configuration: ck.0, options: options, kernel: ck.1, initImage: ck.2)
 
             if !self.managementFlags.cidfile.isEmpty {
                 let path = self.managementFlags.cidfile

Sources/ContainerCommands/Container/ContainerRun.swift

@@ -113,7 +113,8 @@ extension Application {
             try await client.create(
                 configuration: ck.0,
                 options: options,
-                kernel: ck.1
+                kernel: ck.1,
+                initImage: ck.2
             )
 
             let detach = self.managementFlags.detach

Sources/Services/ContainerAPIService/Client/ContainerClient.swift

@@ -48,7 +48,8 @@ public struct ContainerClient: Sendable {
     public func create(
         configuration: ContainerConfiguration,
         options: ContainerCreateOptions = .default,
-        kernel: Kernel
+        kernel: Kernel,
+        initImage: String? = nil
     ) async throws {
         do {
             let request = XPCMessage(route: .containerCreate)
@@ -60,6 +61,10 @@ public struct ContainerClient: Sendable {
             request.set(key: .kernel, value: kdata)
             request.set(key: .containerOptions, value: odata)
 
+            if let initImage {
+                request.set(key: .initImage, value: initImage)
+            }
+
             try await xpcSend(message: request)
         } catch {
             throw ContainerizationError(

Sources/Services/ContainerAPIService/Client/Flags.swift

@@ -158,6 +158,12 @@ public struct Flags {
         )
         public var kernel: String?
 
+        @Option(
+            name: .long,
+            help: .init("Use a custom init image instead of the default", valueName: "image")
+        )
+        public var initImage: String?
+
         @Option(name: [.short, .customLong("label")], help: "Add a key=value label to the container")
         public var labels: [String] = []
 

Sources/Services/ContainerAPIService/Client/Utility.swift

@@ -84,7 +84,7 @@ public struct Utility {
         imageFetch: Flags.ImageFetch,
         progressUpdate: @escaping ProgressUpdateHandler,
         log: Logger
-    ) async throws -> (ContainerConfiguration, Kernel) {
+    ) async throws -> (ContainerConfiguration, Kernel, String?) {
         var requestedPlatform = Parser.platform(os: management.os, arch: management.arch)
         // Prefer --platform
         if let platform = management.platform {
@@ -129,8 +129,9 @@ public struct Utility {
             .setItemsName("blobs"),
         ])
         let fetchInitTask = await taskManager.startTask()
+        let initImageRef = management.initImage ?? ClientImage.initImageRef
         let initImage = try await ClientImage.fetch(
-            reference: ClientImage.initImageRef, platform: .current, scheme: scheme,
+            reference: initImageRef, platform: .current, scheme: scheme,
             progressUpdate: ProgressTaskCoordinator.handler(for: fetchInitTask, from: progressUpdate),
             maxConcurrentDownloads: imageFetch.maxConcurrentDownloads)
 
@@ -252,7 +253,7 @@ public struct Utility {
             config.runtimeHandler = runtime
         }
 
-        return (config, kernel)
+        return (config, kernel, management.initImage)
     }
 
     static func getAttachmentConfigurations(

Sources/Services/ContainerAPIService/Client/XPC+.swift

@@ -106,6 +106,9 @@ public enum XPCKeys: String {
     case systemPlatform
     case kernelForce
 
+    /// Init image reference
+    case initImage
+
     /// Volume
     case volume
     case volumes

Sources/Services/ContainerAPIService/Server/Containers/ContainersHarness.swift

@@ -187,7 +187,9 @@ public struct ContainersHarness: Sendable {
         let config = try JSONDecoder().decode(ContainerConfiguration.self, from: data)
         let kernel = try JSONDecoder().decode(Kernel.self, from: kdata)
 
-        try await service.create(configuration: config, kernel: kernel, options: options)
+        let initImage = message.string(key: .initImage)
+
+        try await service.create(configuration: config, kernel: kernel, options: options, initImage: initImage)
         return message.reply()
     }
 

Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift

@@ -192,7 +192,7 @@ public actor ContainersService {
     }
 
     /// Create a new container from the provided id and configuration.
-    public func create(configuration: ContainerConfiguration, kernel: Kernel, options: ContainerCreateOptions) async throws {
+    public func create(configuration: ContainerConfiguration, kernel: Kernel, options: ContainerCreateOptions, initImage: String? = nil) async throws {
         self.log.debug("\(#function)")
 
         try await self.lock.withLock { context in
@@ -233,11 +233,14 @@ public actor ContainersService {
 
             let path = self.containerRoot.appendingPathComponent(configuration.id)
             let systemPlatform = kernel.platform
-            let initFs = try await self.getInitBlock(for: systemPlatform.ociPlatform())
+
+            // Fetch init image (custom or default)
+            self.log.info("Using init image: \(initImage ?? ClientImage.initImageRef)")
+            let initFilesystem = try await self.getInitBlock(for: systemPlatform.ociPlatform(), imageRef: initImage)
 
             let bundle = try ContainerResource.Bundle.create(
                 path: path,
-                initialFilesystem: initFs,
+                initialFilesystem: initFilesystem,
                 kernel: kernel,
                 containerConfiguration: configuration
             )
@@ -622,8 +625,9 @@ public actor ContainersService {
         return options
     }
 
-    private func getInitBlock(for platform: Platform) async throws -> Filesystem {
-        let initImage = try await ClientImage.fetch(reference: ClientImage.initImageRef, platform: platform)
+    private func getInitBlock(for platform: Platform, imageRef: String? = nil) async throws -> Filesystem {
+        let ref = imageRef ?? ClientImage.initImageRef
+        let initImage = try await ClientImage.fetch(reference: ref, platform: platform)
         var fs = try await initImage.getCreateSnapshot(platform: platform)
         fs.options = ["ro"]
         return fs

Tests/CLITests/Subcommands/Run/TestCLIRunInitImage.swift

@@ -0,0 +1,122 @@
+//===----------------------------------------------------------------------===//
+// Copyright © 2026 Apple Inc. and the container project authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//===----------------------------------------------------------------------===//
+
+import Foundation
+import Testing
+
+/// Tests for the `--init-image` flag which allows specifying a custom init filesystem
+/// image for microvms. This enables customizing boot-time behavior before the OCI
+/// container starts.
+///
+/// See: https://github.com/apple/container/discussions/838
+///
+/// Note: A full integration test that verifies custom init behavior would require
+/// a pre-built test init image that writes a marker to /dev/kmsg. This can be added
+/// once a test init image is published to the registry.
+class TestCLIRunInitImage: CLITest {
+    private func getTestName() -> String {
+        Test.current!.name.trimmingCharacters(in: ["(", ")"]).lowercased()
+    }
+
+    /// Test that specifying a non-existent init-image fails with an appropriate error.
+    @Test func testRunWithNonExistentInitImage() throws {
+        let name = getTestName()
+        let nonExistentImage = "nonexistent.invalid/init-image:does-not-exist"
+
+        #expect(throws: CLIError.self, "expected container run with non-existent init-image to fail") {
+            let (_, _, error, status) = try run(arguments: [
+                "run",
+                "--rm",
+                "--name", name,
+                "-d",
+                "--init-image", nonExistentImage,
+                alpine,
+                "sleep", "infinity",
+            ])
+            defer { try? doRemove(name: name, force: true) }
+            if status != 0 {
+                throw CLIError.executionFailed("command failed: \(error)")
+            }
+        }
+    }
+
+    /// Test that the `--init-image` flag is recognized and documented in CLI help.
+    @Test func testInitImageFlagInHelp() throws {
+        let (_, output, _, status) = try run(arguments: ["run", "--help"])
+        #expect(status == 0, "expected help command to succeed")
+        #expect(
+            output.contains("--init-image"),
+            "expected help output to contain --init-image flag"
+        )
+        #expect(
+            output.contains("custom init image"),
+            "expected help output to describe the init-image flag"
+        )
+    }
+
+    /// Test that the `--init-image` flag works with `container create` command.
+    @Test func testCreateWithNonExistentInitImage() throws {
+        let name = getTestName()
+        let nonExistentImage = "nonexistent.invalid/init-image:does-not-exist"
+
+        #expect(throws: CLIError.self, "expected container create with non-existent init-image to fail") {
+            let (_, _, error, status) = try run(arguments: [
+                "create",
+                "--rm",
+                "--name", name,
+                "--init-image", nonExistentImage,
+                alpine,
+                "echo", "hello",
+            ])
+            defer { try? doRemove(name: name, force: true) }
+            if status != 0 {
+                throw CLIError.executionFailed("command failed: \(error)")
+            }
+        }
+    }
+
+    /// Test that explicitly specifying the default init image works the same as
+    /// not specifying any init image.
+    @Test func testRunWithExplicitDefaultInitImage() throws {
+        let name = getTestName()
+
+        // Get the default init image reference
+        let (_, defaultInitImage, _, propStatus) = try run(arguments: [
+            "system", "property", "get", "image.init",
+        ])
+
+        guard propStatus == 0 else {
+            print("Skipping testRunWithExplicitDefaultInitImage: could not get default init image")
+            return
+        }
+
+        let initImage = defaultInitImage.trimmingCharacters(in: .whitespacesAndNewlines)
+
+        // Run container with explicit default init image
+        try doLongRun(name: name, args: ["--init-image", initImage])
+        defer {
+            try? doStop(name: name)
+        }
+
+        // Verify container is running and functional
+        try waitForContainerRunning(name)
+        let output = try doExec(name: name, cmd: ["echo", "hello"])
+        #expect(
+            output.trimmingCharacters(in: .whitespacesAndNewlines) == "hello",
+            "expected 'hello' output from exec, got '\(output)'"
+        )
+    }
+}

docs/command-reference.md

@@ -50,6 +50,7 @@ container run [<options>] <image> [<arguments> ...]
 *   `--dns-option <option>`: DNS options
 *   `--dns-search <domain>`: DNS search domains
 *   `--entrypoint <cmd>`: Override the entrypoint of the image
+*   `--init-image <image>`: Use a custom init image instead of the default. This allows customizing boot-time behavior before the OCI container starts, such as running VM-level daemons, configuring eBPF filters, or debugging the init process.
 *   `-k, --kernel <path>`: Set a custom kernel path
 *   `-l, --label <label>`: Add a key=value label to the container
 *   `--mount <mount>`: Add a mount to the container (format: type=<>,source=<>,target=<>,readonly)
@@ -61,6 +62,7 @@ container run [<options>] <image> [<arguments> ...]
 *   `--platform <platform>`: Platform for the image if it's multi-platform. This takes precedence over --os and --arch
 *   `--publish-socket <spec>`: Publish a socket from container to host (format: host_path:container_path)
 *   `--rm, --remove`: Remove the container after it stops
+*   `--rosetta`: Enable Rosetta in the container
 *   `--ssh`: Forward SSH agent socket to container
 *   `--tmpfs <tmpfs>`: Add a tmpfs mount to the container at the given path
 *   `-v, --volume <volume>`: Bind mount a volume into the container
@@ -101,6 +103,9 @@ container run -e NODE_ENV=production --cpus 2 --memory 1G node:18
 
 # run a container with a specific MAC address
 container run --network default,mac=02:42:ac:11:00:02 ubuntu:latest
+
+# run a container with a custom init image for boot customization
+container run --init-image local/custom-init:latest ubuntu:latest
 ```
 
 ### `container build`
@@ -200,6 +205,7 @@ container create [<options>] <image> [<arguments> ...]
 *   `--dns-option <option>`: DNS options
 *   `--dns-search <domain>`: DNS search domains
 *   `--entrypoint <cmd>`: Override the entrypoint of the image
+*   `--init-image <image>`: Use a custom init image instead of the default. This allows customizing boot-time behavior before the OCI container starts, such as running VM-level daemons, configuring eBPF filters, or debugging the init process.
 *   `-k, --kernel <path>`: Set a custom kernel path
 *   `-l, --label <label>`: Add a key=value label to the container
 *   `--mount <mount>`: Add a mount to the container (format: type=<>,source=<>,target=<>,readonly)
@@ -211,6 +217,7 @@ container create [<options>] <image> [<arguments> ...]
 *   `--platform <platform>`: Platform for the image if it's multi-platform. This takes precedence over --os and --arch
 *   `--publish-socket <spec>`: Publish a socket from container to host (format: host_path:container_path)
 *   `--rm, --remove`: Remove the container after it stops
+*   `--rosetta`: Enable Rosetta in the container
 *   `--ssh`: Forward SSH agent socket to container
 *   `--tmpfs <tmpfs>`: Add a tmpfs mount to the container at the given path
 *   `-v, --volume <volume>`: Bind mount a volume into the container

docs/how-to.md

@@ -497,6 +497,82 @@ container run --name nested-virtualization --virtualization --kernel /path/to/a/
 [    0.017893] kvm [1]: Hyp mode initialized successfully
 ```
 
+## Use a custom init image
+
+The `--init-image` flag allows you to specify a custom init filesystem image for the lightweight VM that runs your container. This enables:
+
+- Custom boot-time logic before the OCI container starts
+- Running additional processes and daemons (e.g., eBPF network filters, logging agents) inside the VM
+- Debugging or instrumenting the init process
+
+### Create a custom init image
+
+A custom init image wraps the default `vminitd` binary, allowing you to run custom logic before handing off to the standard init process.
+
+**1. Create a wrapper binary (example in Go for easy cross-compilation):**
+
+```go
+// wrapper.go
+package main
+
+import (
+    "os"
+    "syscall"
+)
+
+func main() {
+    // Write a message to kernel log
+    kmsg, err := os.OpenFile("/dev/kmsg", os.O_WRONLY, 0)
+    if err == nil {
+        kmsg.WriteString("<6>custom-init: === CUSTOM INIT IMAGE RUNNING ===\n")
+        kmsg.Close()
+    }
+
+    // Execute the real vminitd
+    err = syscall.Exec("/sbin/vminitd.real", os.Args, os.Environ())
+    if err != nil {
+        os.Exit(1)
+    }
+}
+```
+
+**2. Build the wrapper for Linux arm64:**
+
+```bash
+CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o wrapper wrapper.go
+```
+
+**3. Create a Containerfile:**
+
+```dockerfile
+FROM ghcr.io/apple/containerization/vminit:latest AS base
+
+FROM ghcr.io/apple/containerization/vminit:latest
+COPY --from=base /sbin/vminitd /sbin/vminitd.real
+COPY wrapper /sbin/vminitd
+```
+
+**4. Build the custom init image:**
+
+```bash
+container build -t local/custom-init:latest .
+```
+
+### Run a container with a custom init image
+
+```bash
+container run --name my-container --init-image local/custom-init:latest alpine:latest echo "hello"
+```
+
+### Verify the custom init is running
+
+Check the VM boot logs to confirm your custom init code executed:
+
+```console
+% container logs --boot my-container | grep custom-init
+[    0.129230] custom-init: === CUSTOM INIT IMAGE RUNNING ===
+```
+
 ## Configure system properties
 
 The `container system property` subcommand manages the configuration settings for the `container` CLI and services. You can customize various aspects of container behavior, including build settings, default images, and network configuration.