vminitd: Proper pty setup / mount /dev/console (#248)

Sources/Integration/Suite.swift

@@ -215,6 +215,7 @@ struct IntegrationSuite: AsyncParsableCommand {
             "nested virt": testNestedVirtualizationEnabled,
             "container manager": testContainerManagerCreate,
             "container reuse": testContainerReuse,
+            "container /dev/console": testContainerDevConsole,
         ]
 
         var passed = 0

Sources/Integration/VMTests.swift

@@ -257,6 +257,54 @@ extension IntegrationSuite {
         }
     }
 
+    func testContainerDevConsole() async throws {
+        let id = "test-container-devconsole"
+
+        let bs = try await bootstrap()
+
+        let manager = try ContainerManager(vmm: bs.vmm)
+        defer {
+            try? manager.delete(id)
+        }
+
+        let buffer = BufferWriter()
+        let container = try await manager.create(
+            id,
+            image: bs.image,
+            rootfs: bs.rootfs
+        ) { config in
+            // We mount devtmpfs by default, and while this includes creating
+            // /dev/console typically that'll be pointing to /dev/hvc0 (the
+            // virtio serial console). This is just a character device, so a trivial
+            // way to check that our bind mounted console setup worked is by just
+            // parsing `mount`'s output and looking for /dev/console as it wouldn't
+            // be there normally without our dance.
+            config.process.arguments = ["mount"]
+            config.process.terminal = true
+            config.process.stdout = buffer
+        }
+
+        try await container.create()
+        try await container.start()
+
+        let status = try await container.wait()
+        try await container.stop()
+        guard status == 0 else {
+            throw IntegrationError.assert(msg: "process status \(status) != 0")
+        }
+
+        guard let str = String(data: buffer.data, encoding: .utf8) else {
+            throw IntegrationError.assert(
+                msg: "failed to convert standard output to a UTF8 string")
+        }
+
+        let devConsole = "/dev/console"
+        guard str.contains(devConsole) else {
+            throw IntegrationError.assert(
+                msg: "process should have \(devConsole) in `mount` output")
+        }
+    }
+
     private func createMountDirectory() throws -> URL {
         let dir = FileManager.default.uniqueTemporaryDirectory(create: true)
         try "hello".write(to: dir.appendingPathComponent("hi.txt"), atomically: true, encoding: .utf8)

Sources/cctl/RunCommand.swift

@@ -68,7 +68,8 @@ extension Application {
         @Option(name: .long, help: "Current working directory")
         var cwd: String = "/"
 
-        @Argument var arguments: [String] = ["/bin/sh"]
+        @Argument(parsing: .captureForPassthrough)
+        var arguments: [String] = ["/bin/sh"]
 
         func run() async throws {
             let kernel = Kernel(

vminitd/Sources/vmexec/Console.swift

@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+// Copyright © 2025 Apple Inc. and the Containerization project authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//===----------------------------------------------------------------------===//
+
+import Foundation
+import Musl
+
+class Console {
+    let master: Int32
+    let slavePath: String
+
+    init() throws {
+        let masterFD = open("/dev/ptmx", O_RDWR | O_NOCTTY | O_CLOEXEC)
+        guard masterFD != -1 else {
+            throw App.Errno(stage: "open_ptmx")
+        }
+
+        guard unlockpt(masterFD) == 0 else {
+            throw App.Errno(stage: "unlockpt")
+        }
+
+        guard let slavePath = ptsname(masterFD) else {
+            throw App.Errno(stage: "ptsname")
+        }
+
+        self.master = masterFD
+        self.slavePath = String(cString: slavePath)
+    }
+
+    func configureStdIO() throws {
+        let path = self.slavePath
+        let slaveFD = open(path, O_RDWR)
+        guard slaveFD != -1 else {
+            throw App.Errno(stage: "open_pts")
+        }
+        defer { Musl.close(slaveFD) }
+
+        for fd: Int32 in 0...2 {
+            guard dup3(slaveFD, fd, 0) != -1 else {
+                throw App.Errno(stage: "dup3")
+            }
+        }
+    }
+
+    func close() throws {
+        guard Musl.close(self.master) == 0 else {
+            throw App.Errno(stage: "close")
+        }
+    }
+}

vminitd/Sources/vmexec/ExecCommand.swift

@@ -62,37 +62,68 @@ struct ExecCommand: ParsableCommand {
         process: ContainerizationOCI.Process,
         log: Logger
     ) throws {
-        // CLOEXEC the pipe fd that signals process readiness.
-        let syncfd = FileHandle(fileDescriptor: 3)
-        if fcntl(3, F_SETFD, FD_CLOEXEC) == -1 {
-            throw App.Errno(stage: "cloexec(syncfd)")
-        }
+        let syncPipe = FileHandle(fileDescriptor: 3)
+        let ackPipe = FileHandle(fileDescriptor: 4)
 
         try Self.enterNS(path: "/proc/\(self.parentPid)/ns/cgroup", nsType: CLONE_NEWCGROUP)
         try Self.enterNS(path: "/proc/\(self.parentPid)/ns/pid", nsType: CLONE_NEWPID)
         try Self.enterNS(path: "/proc/\(self.parentPid)/ns/uts", nsType: CLONE_NEWUTS)
         try Self.enterNS(path: "/proc/\(self.parentPid)/ns/mnt", nsType: CLONE_NEWNS)
 
-        let childPipe = Pipe()
-        try childPipe.setCloexec()
         let processID = fork()
 
         guard processID != -1 else {
-            try? childPipe.fileHandleForReading.close()
-            try? childPipe.fileHandleForWriting.close()
-            try? syncfd.close()
+            try? syncPipe.close()
+            try? ackPipe.close()
 
             throw App.Errno(stage: "fork")
         }
 
         if processID == 0 {  // child
-            try childPipe.fileHandleForReading.close()
-            try syncfd.close()
+            // Wait for the grandparent to tell us that they acked our pid.
+            guard let data = try ackPipe.read(upToCount: App.ackPid.count) else {
+                throw App.Failure(message: "read ack pipe")
+            }
+            guard let pidAckStr = String(data: data, encoding: .utf8) else {
+                throw App.Failure(message: "convert ack pipe data to string")
+            }
+
+            guard pidAckStr == App.ackPid else {
+                throw App.Failure(message: "received invalid acknowledgement string: \(pidAckStr)")
+            }
 
             guard setsid() != -1 else {
                 throw App.Errno(stage: "setsid()")
             }
 
+            if process.terminal {
+                let pty = try Console()
+                try pty.configureStdIO()
+                var masterFD = pty.master
+
+                let data = Data(bytes: &masterFD, count: MemoryLayout.size(ofValue: masterFD))
+                try syncPipe.write(contentsOf: data)
+                try syncPipe.close()
+
+                // Wait for the grandparent to tell us that they acked our console.
+                guard let data = try ackPipe.read(upToCount: App.ackConsole.count) else {
+                    throw App.Failure(message: "read ack pipe")
+                }
+
+                guard let consoleAckStr = String(data: data, encoding: .utf8) else {
+                    throw App.Failure(message: "convert ack pipe data to string")
+                }
+
+                guard consoleAckStr == App.ackConsole else {
+                    throw App.Failure(message: "received invalid acknowledgement string: \(consoleAckStr)")
+                }
+
+                guard ioctl(0, UInt(TIOCSCTTY), 0) != -1 else {
+                    throw App.Errno(stage: "setctty(0)")
+                }
+                try pty.close()
+            }
+
             // Apply O_CLOEXEC to all file descriptors except stdio.
             // This ensures that all unwanted fds we may have accidentally
             // inherited are marked close-on-exec so they stay out of the
@@ -106,26 +137,13 @@ struct ExecCommand: ParsableCommand {
             // Set uid, gid, and supplementary groups
             try App.setPermissions(user: process.user)
 
-            if process.terminal {
-                guard ioctl(0, UInt(TIOCSCTTY), 0) != -1 else {
-                    throw App.Errno(stage: "setctty()")
-                }
-            }
-
             try App.exec(process: process)
         } else {  // parent process
-            try childPipe.fileHandleForWriting.close()
-
-            // wait until the pipe is closed then carry on.
-            _ = try childPipe.fileHandleForReading.readToEnd()
-            try childPipe.fileHandleForReading.close()
-
-            // send our child's pid to our parent before we exit.
+            // Send our child's pid to our parent before we exit.
             var childPid = processID
             let data = Data(bytes: &childPid, count: MemoryLayout.size(ofValue: childPid))
 
-            try syncfd.write(contentsOf: data)
-            try syncfd.close()
+            try syncPipe.write(contentsOf: data)
         }
     }
 }

vminitd/Sources/vmexec/Mount.swift

@@ -36,8 +36,7 @@ struct ContainerMount {
     }
 
     func configureConsole() throws {
-        let ptmx = self.rootfs.standardizingPath.appendingPathComponent("/dev/ptmx")
-
+        let ptmx = self.rootfs.standardizingPath.appendingPathComponent("dev/ptmx")
         guard remove(ptmx) == 0 else {
             throw App.Errno(stage: "remove(ptmx)")
         }

vminitd/Sources/vmexec/RunCommand.swift

@@ -49,92 +49,120 @@ struct RunCommand: ParsableCommand {
         try reOpenDevNull()
     }
 
-    private func execInNamespace(spec: ContainerizationOCI.Spec, log: Logger) throws {
+    private func childSetup(
+        spec: ContainerizationOCI.Spec,
+        ackPipe: FileHandle,
+        syncPipe: FileHandle,
+        log: Logger
+    ) throws {
         guard let process = spec.process else {
-            fatalError("no process configuration found in runtime spec")
+            throw App.Failure(message: "no process configuration found in runtime spec")
         }
         guard let root = spec.root else {
-            fatalError("no root found in runtime spec")
+            throw App.Failure(message: "no root found in runtime spec")
         }
 
-        let syncfd = FileHandle(fileDescriptor: 3)
-        if fcntl(3, F_SETFD, FD_CLOEXEC) == -1 {
-            throw App.Errno(stage: "cloexec(syncfd)")
+        // Wait for the grandparent to tell us that they acked our pid.
+        guard let data = try ackPipe.read(upToCount: App.ackPid.count) else {
+            throw App.Failure(message: "read ack pipe")
         }
+        guard let pidAckStr = String(data: data, encoding: .utf8) else {
+            throw App.Failure(message: "convert ack pipe data to string")
+        }
+
+        guard pidAckStr == App.ackPid else {
+            throw App.Failure(message: "received invalid acknowledgement string: \(pidAckStr)")
+        }
+
+        guard unshare(CLONE_NEWCGROUP) == 0 else {
+            throw App.Errno(stage: "unshare(cgroup)")
+        }
+
+        guard setsid() != -1 else {
+            throw App.Errno(stage: "setsid()")
+        }
+
+        try childRootSetup(rootfs: root, mounts: spec.mounts, log: log)
+
+        if process.terminal {
+            let pty = try Console()
+            try pty.configureStdIO()
+            var masterFD = pty.master
+
+            let data = Data(bytes: &masterFD, count: MemoryLayout.size(ofValue: masterFD))
+            try syncPipe.write(contentsOf: data)
+            try syncPipe.close()
+
+            // Wait for the grandparent to tell us that they acked our console.
+            guard let data = try ackPipe.read(upToCount: App.ackConsole.count) else {
+                throw App.Failure(message: "read ack pipe")
+            }
+
+            guard let consoleAckStr = String(data: data, encoding: .utf8) else {
+                throw App.Failure(message: "convert ack pipe data to string")
+            }
+
+            guard consoleAckStr == App.ackConsole else {
+                throw App.Failure(message: "received invalid acknowledgement string: \(consoleAckStr)")
+            }
+
+            guard ioctl(0, UInt(TIOCSCTTY), 0) != -1 else {
+                throw App.Errno(stage: "setctty(0)")
+            }
+
+            try mountConsole(path: pty.slavePath)
+            try pty.close()
+        }
+
+        if !spec.hostname.isEmpty {
+            let errCode = spec.hostname.withCString { ptr in
+                Musl.sethostname(ptr, spec.hostname.count)
+            }
+            guard errCode == 0 else {
+                throw App.Errno(stage: "sethostname()")
+            }
+        }
+
+        // Apply O_CLOEXEC to all file descriptors except stdio.
+        // This ensures that all unwanted fds we may have accidentally
+        // inherited are marked close-on-exec so they stay out of the
+        // container.
+        try App.applyCloseExecOnFDs()
+
+        try App.setRLimits(rlimits: process.rlimits)
+
+        // Change stdio to be owned by the requested user.
+        try App.fixStdioPerms(user: process.user)
+
+        // Set uid, gid, and supplementary groups.
+        try App.setPermissions(user: process.user)
+
+        // Finally execve the container process.
+        try App.exec(process: process)
+    }
+
+    private func execInNamespace(spec: ContainerizationOCI.Spec, log: Logger) throws {
+        let syncPipe = FileHandle(fileDescriptor: 3)
+        let ackPipe = FileHandle(fileDescriptor: 4)
 
         guard unshare(CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWUTS) == 0 else {
             throw App.Errno(stage: "unshare(pid|mnt|uts)")
         }
 
-        let childPipe = Pipe()
-        try childPipe.setCloexec()
         let processID = fork()
-
         guard processID != -1 else {
-            try? childPipe.fileHandleForReading.close()
-            try? childPipe.fileHandleForWriting.close()
-            try? syncfd.close()
-
+            try? syncPipe.close()
+            try? ackPipe.close()
             throw App.Errno(stage: "fork")
         }
 
         if processID == 0 {  // child
-            try childPipe.fileHandleForReading.close()
-            try syncfd.close()
-
-            guard unshare(CLONE_NEWCGROUP) == 0 else {
-                throw App.Errno(stage: "unshare(cgroup)")
-            }
-
-            guard setsid() != -1 else {
-                throw App.Errno(stage: "setsid()")
-            }
-
-            try childRootSetup(rootfs: root, mounts: spec.mounts, log: log)
-
-            if !spec.hostname.isEmpty {
-                let errCode = spec.hostname.withCString { ptr in
-                    Musl.sethostname(ptr, spec.hostname.count)
-                }
-                guard errCode == 0 else {
-                    throw App.Errno(stage: "sethostname()")
-                }
-            }
-
-            // Apply O_CLOEXEC to all file descriptors except stdio.
-            // This ensures that all unwanted fds we may have accidentally
-            // inherited are marked close-on-exec so they stay out of the
-            // container.
-            try App.applyCloseExecOnFDs()
-
-            try App.setRLimits(rlimits: process.rlimits)
-
-            // Change stdio to be owned by the requested user.
-            try App.fixStdioPerms(user: process.user)
-
-            // Set uid, gid, and supplementary groups.
-            try App.setPermissions(user: process.user)
-
-            if process.terminal {
-                guard ioctl(0, UInt(TIOCSCTTY), 0) != -1 else {
-                    throw App.Errno(stage: "setctty()")
-                }
-            }
-
-            try App.exec(process: process)
+            try childSetup(spec: spec, ackPipe: ackPipe, syncPipe: syncPipe, log: log)
         } else {  // parent process
-            try childPipe.fileHandleForWriting.close()
-
-            // wait until the pipe is closed then carry on.
-            _ = try childPipe.fileHandleForReading.readToEnd()
-            try childPipe.fileHandleForReading.close()
-
-            // send our child's pid to our parent before we exit.
+            // Send our child's pid before we exit.
             var childPid = processID
             let data = Data(bytes: &childPid, count: MemoryLayout.size(ofValue: childPid))
-
-            try syncfd.write(contentsOf: data)
-            try syncfd.close()
+            try syncPipe.write(contentsOf: data)
         }
     }
 
@@ -222,7 +250,6 @@ struct RunCommand: ParsableCommand {
         if newRoot <= 0 {
             throw App.Errno(stage: "open(newroot)")
         }
-
         defer { close(newRoot) }
 
         // change cwd to the new root
@@ -257,4 +284,19 @@ struct RunCommand: ParsableCommand {
         _ = cStringCopy.initialize(from: cString)
         return UnsafeMutablePointer(cStringCopy.baseAddress)
     }
+
+    private func mountConsole(path: String) throws {
+        let console = "/dev/console"
+        if access(console, F_OK) != 0 {
+            let fd = open(console, O_RDWR | O_CREAT, mode_t(UInt16(0o600)))
+            guard fd != -1 else {
+                throw App.Errno(stage: "open(/dev/console)")
+            }
+            close(fd)
+        }
+
+        guard mount(path, console, "bind", UInt(MS_BIND), nil) == 0 else {
+            throw App.Errno(stage: "mount(console)")
+        }
+    }
 }

vminitd/Sources/vmexec/vmexec.swift

@@ -29,6 +29,9 @@ import Musl
 
 @main
 struct App: ParsableCommand {
+    static let ackPid = "AckPid"
+    static let ackConsole = "AckConsole"
+
     static let configuration = CommandConfiguration(
         commandName: "vmexec",
         version: "0.1.0",
@@ -161,4 +164,11 @@ extension App {
         let posix = POSIXError(.init(rawValue: errno)!, userInfo: ["stage": stage])
         return ContainerizationError(.internalError, message: "\(info) \(String(describing: posix))")
     }
+
+    static func Failure(message: String) -> ContainerizationError {
+        ContainerizationError(
+            .internalError,
+            message: message
+        )
+    }
 }

vminitd/Sources/vminitd/IOPair.swift

@@ -21,104 +21,162 @@ import Logging
 import Synchronization
 
 final class IOPair: Sendable {
-    let readFrom: IOCloser
-    let writeTo: IOCloser
-    nonisolated(unsafe) let buffer: UnsafeMutableBufferPointer<UInt8>
+    private let io: Mutex<IO>
     private let logger: Logger?
+    private let reason: String
 
-    private let done: Atomic<Bool>
+    private struct IO {
+        let from: IOCloser
+        let to: IOCloser
+        let buffer: UnsafeMutableBufferPointer<UInt8>
+        var closed: Bool
+        let logger: Logger?
 
-    init(readFrom: IOCloser, writeTo: IOCloser, logger: Logger? = nil) {
-        self.readFrom = readFrom
-        self.writeTo = writeTo
-        self.done = Atomic(false)
-        self.buffer = UnsafeMutableBufferPointer<UInt8>.allocate(capacity: Int(getpagesize()))
-        self.logger = logger
-    }
+        func drain() {
+            let readFrom = OSFile(fd: from.fileDescriptor)
+            let writeTo = OSFile(fd: to.fileDescriptor)
 
-    func relay() throws {
-        let readFromFd = self.readFrom.fileDescriptor
-        let writeToFd = self.writeTo.fileDescriptor
-
-        let readFrom = OSFile(fd: readFromFd)
-        let writeTo = OSFile(fd: writeToFd)
-
-        try ProcessSupervisor.default.poller.add(readFromFd, mask: EPOLLIN) { mask in
-            if mask.isHangup && !mask.readyToRead {
-                self.close()
-                return
-            }
-            // Loop so that in the case that someone wrote > buf.count down the pipe
-            // we properly will drain it fully.
             while true {
-                let r = readFrom.read(self.buffer)
+                let r = readFrom.read(buffer)
                 if r.read > 0 {
                     let view = UnsafeMutableBufferPointer(
-                        start: self.buffer.baseAddress,
+                        start: buffer.baseAddress,
                         count: r.read
                     )
 
                     let w = writeTo.write(view)
                     if w.wrote != r.read {
-                        self.logger?.error("stopping relay: short write for stdio")
-                        self.close()
                         return
                     }
                 }
 
                 switch r.action {
-                case .error(let errno):
-                    self.logger?.error("failed with errno \(errno) while reading for fd \(readFromFd)")
-                    fallthrough
-                case .eof:
-                    self.close()
-                    self.logger?.debug("closing relay for \(readFromFd)")
-                    return
-                case .again:
-                    // We read all we could, exit.
-                    if mask.isHangup {
-                        self.close()
-                    }
+                case .eof, .again, .error(_):
                     return
                 default:
                     break
                 }
             }
         }
+
+        mutating func close() {
+            if self.closed {
+                return
+            }
+
+            // Try and drain IO first.
+            self.drain()
+
+            // Remove the fd from our global epoll instance first.
+            let readFromFd = self.from.fileDescriptor
+            do {
+                try ProcessSupervisor.default.poller.delete(readFromFd)
+            } catch {
+                self.logger?.error("failed to delete fd from epoll \(readFromFd): \(error)")
+            }
+
+            do {
+                try self.from.close()
+            } catch {
+                self.logger?.error("failed to close reader fd for IOPair: \(error)")
+            }
+
+            do {
+                try self.to.close()
+            } catch {
+                self.logger?.error("failed to close writer fd for IOPair: \(error)")
+            }
+            self.buffer.deallocate()
+            self.closed = true
+        }
+    }
+
+    init(
+        readFrom: IOCloser,
+        writeTo: IOCloser,
+        reason: String,
+        logger: Logger? = nil
+    ) {
+        let buffer = UnsafeMutableBufferPointer<UInt8>.allocate(capacity: Int(getpagesize()))
+        self.io = Mutex(
+            IO(
+                from: readFrom,
+                to: writeTo,
+                buffer: buffer,
+                closed: false,
+                logger: logger
+            ))
+        self.reason = reason
+        self.logger = logger
+    }
+
+    func relay(ignoreHup: Bool = false) throws {
+        self.logger?.info("setting up relay for \(reason)")
+
+        let (readFromFd, writeToFd) = self.io.withLock { io in
+            (io.from.fileDescriptor, io.to.fileDescriptor)
+        }
+
+        let readFrom = OSFile(fd: readFromFd)
+        let writeTo = OSFile(fd: writeToFd)
+
+        try ProcessSupervisor.default.poller.add(readFromFd, mask: EPOLLIN) { mask in
+            self.io.withLock { io in
+                if io.closed {
+                    return
+                }
+
+                if mask.isHangup && !mask.readyToRead {
+                    self.logger?.debug("received EPOLLHUP with no EPOLLIN")
+                    if !ignoreHup {
+                        io.close()
+                    }
+                    return
+                }
+
+                // Loop so we drain fully.
+                while true {
+                    let r = readFrom.read(io.buffer)
+                    if r.read > 0 {
+                        let view = UnsafeMutableBufferPointer(
+                            start: io.buffer.baseAddress,
+                            count: r.read
+                        )
+
+                        let w = writeTo.write(view)
+                        if w.wrote != r.read {
+                            self.logger?.error("stopping relay: short write for stdio")
+                            io.close()
+                            return
+                        }
+                    }
+
+                    switch r.action {
+                    case .error(let errno):
+                        self.logger?.error("failed with errno \(errno) while reading for fd \(readFromFd)")
+                        fallthrough
+                    case .eof:
+                        self.logger?.debug("closing relay for \(readFromFd)")
+                        io.close()
+                        return
+                    case .again:
+                        if mask.isHangup && !ignoreHup {
+                            self.logger?.error("received EPOLLHUP and EAGAIN exiting")
+                            self.close()
+                        }
+                        return
+                    default:
+                        break
+                    }
+                }
+            }
+        }
     }
 
     func close() {
-        guard
-            self.done.compareExchange(
-                expected: false,
-                desired: true,
-                successOrdering: .acquiringAndReleasing,
-                failureOrdering: .acquiring
-            ).exchanged
-        else {
-            return
-        }
-
-        self.buffer.deallocate()
-
-        let readFromFd = self.readFrom.fileDescriptor
-        // Remove the fd from our global epoll instance first.
-        do {
-            try ProcessSupervisor.default.poller.delete(readFromFd)
-        } catch {
-            self.logger?.error("failed to delete fd from epoll \(readFromFd): \(error)")
-        }
-
-        do {
-            try self.readFrom.close()
-        } catch {
-            self.logger?.error("failed to close reader fd for IOPair: \(error)")
-        }
-
-        do {
-            try self.writeTo.close()
-        } catch {
-            self.logger?.error("failed to close writer fd for IOPair: \(error)")
+        self.io.withLock { io in
+            self.logger?.info("closing relay for \(reason)")
+            io.close()
         }
     }
 }

vminitd/Sources/vminitd/ManagedProcess.swift

@@ -29,8 +29,11 @@ final class ManagedProcess: Sendable {
     private let log: Logger
     private let process: Command
     private let state: Mutex<State>
-    private let syncfd: Pipe
     private let owningPid: Int32?
+    private let ackPipe: FileHandle
+    private let syncPipe: FileHandle
+    private let terminal: Bool
+    private let bundle: ContainerizationOCI.Bundle
 
     private struct State {
         init(io: IO) {
@@ -51,10 +54,12 @@ final class ManagedProcess: Sendable {
 
     // swiftlint: disable type_name
     protocol IO {
+        func attach(pid: Int32, fd: Int32) throws
         func start(process: inout Command) throws
-        func closeAfterExec() throws
         func resize(size: Terminal.Size) throws
+        func close() throws
         func closeStdin() throws
+        func closeAfterExec() throws
     }
     // swiftlint: enable type_name
 
@@ -62,6 +67,9 @@ final class ManagedProcess: Sendable {
         log[metadataKey: "id"] = "\(id)"
     }
 
+    private static let ackPid = "AckPid"
+    private static let ackConsole = "AckConsole"
+
     init(
         id: String,
         stdio: HostStdio,
@@ -75,9 +83,13 @@ final class ManagedProcess: Sendable {
         self.log = log
         self.owningPid = owningPid
 
-        let syncfd = Pipe()
-        try syncfd.setCloexec()
-        self.syncfd = syncfd
+        let syncPipe = Pipe()
+        try syncPipe.setCloexec()
+        self.syncPipe = syncPipe.fileHandleForReading
+
+        let ackPipe = Pipe()
+        try ackPipe.setCloexec()
+        self.ackPipe = ackPipe.fileHandleForWriting
 
         let args: [String]
         if let owningPid {
@@ -95,7 +107,10 @@ final class ManagedProcess: Sendable {
         var process = Command(
             "/sbin/vmexec",
             arguments: args,
-            extraFiles: [syncfd.fileHandleForWriting]
+            extraFiles: [
+                syncPipe.fileHandleForWriting,
+                ackPipe.fileHandleForReading,
+            ]
         )
 
         var io: IO
@@ -121,6 +136,8 @@ final class ManagedProcess: Sendable {
         try io.start(process: &process)
 
         self.process = process
+        self.terminal = stdio.terminal
+        self.bundle = bundle
         self.state = Mutex(State(io: io))
     }
 }
@@ -136,30 +153,77 @@ extension ManagedProcess {
 
             // Start the underlying process.
             try process.start()
+            defer {
+                try? self.ackPipe.close()
+                try? self.syncPipe.close()
+            }
 
             // Close our side of any pipes.
-            try syncfd.fileHandleForWriting.close()
             try $0.io.closeAfterExec()
 
-            guard let piddata = try syncfd.fileHandleForReading.readToEnd() else {
+            let size = MemoryLayout<Int32>.size
+            guard let piddata = try syncPipe.read(upToCount: size) else {
                 throw ContainerizationError(.internalError, message: "no pid data from sync pipe")
             }
 
-            let i = piddata.withUnsafeBytes { ptr in
+            guard piddata.count == size else {
+                throw ContainerizationError(.internalError, message: "invalid payload")
+            }
+
+            let pid = piddata.withUnsafeBytes { ptr in
                 ptr.load(as: Int32.self)
             }
 
-            log.info("got back pid data \(i)")
-            $0.pid = i
+            log.info(
+                "got back pid data",
+                metadata: [
+                    "id": "\(pid)"
+                ])
+            $0.pid = pid
+
+            // Ack the pid from the child.
+            log.info(
+                "sending pid acknowledgement",
+                metadata: [
+                    "pid": "\(pid)"
+                ])
+            try self.ackPipe.write(contentsOf: Self.ackPid.data(using: .utf8)!)
+
+            if self.terminal {
+                log.info(
+                    "wait for pty fd",
+                    metadata: [
+                        "id": "\(id)"
+                    ])
+
+                // Wait for a new write that will contain the pty fd if we asked for one.
+                guard let ptyFd = try syncPipe.read(upToCount: size) else {
+                    throw ContainerizationError(
+                        .internalError,
+                        message: "no pty data from sync pipe"
+                    )
+                }
+                let fd = ptyFd.withUnsafeBytes { ptr in
+                    ptr.load(as: Int32.self)
+                }
+                log.info(
+                    "received pty fd from container, attaching",
+                    metadata: [
+                        "id": "\(id)"
+                    ])
+
+                try $0.io.attach(pid: pid, fd: fd)
+                try self.ackPipe.write(contentsOf: Self.ackConsole.data(using: .utf8)!)
+            }
 
             log.info(
                 "started managed process",
                 metadata: [
-                    "pid": "\(i)",
+                    "pid": "\(pid)",
                     "id": "\(id)",
                 ])
 
-            return i
+            return pid
         }
     }
 
@@ -173,6 +237,12 @@ extension ManagedProcess {
 
             $0.exitStatus = status
 
+            do {
+                try $0.io.close()
+            } catch {
+                self.log.error("failed to close io for process: \(error)")
+            }
+
             for waiter in $0.waiters {
                 waiter.resume(returning: status)
             }

vminitd/Sources/vminitd/StandardIO.swift

@@ -43,6 +43,9 @@ final class StandardIO: ManagedProcess.IO & Sendable {
         self.state = Mutex(State())
     }
 
+    // NOP
+    func attach(pid: Int32, fd: Int32) throws {}
+
     func start(process: inout Command) throws {
         try self.state.withLock {
             if let stdinPort = self.hostStdio.stdin {
@@ -60,6 +63,7 @@ final class StandardIO: ManagedProcess.IO & Sendable {
                 let pair = IOPair(
                     readFrom: stdinSocket,
                     writeTo: inPipe.fileHandleForWriting,
+                    reason: "StandardIO stdin",
                     logger: log
                 )
                 $0.stdin = pair
@@ -82,6 +86,7 @@ final class StandardIO: ManagedProcess.IO & Sendable {
                 let pair = IOPair(
                     readFrom: outPipe.fileHandleForReading,
                     writeTo: stdoutSocket,
+                    reason: "StandardIO stdout",
                     logger: log
                 )
                 $0.stdout = pair
@@ -104,6 +109,7 @@ final class StandardIO: ManagedProcess.IO & Sendable {
                 let pair = IOPair(
                     readFrom: errPipe.fileHandleForReading,
                     writeTo: stderrSocket,
+                    reason: "StandardIO stderr",
                     logger: log
                 )
                 $0.stderr = pair
@@ -116,6 +122,25 @@ final class StandardIO: ManagedProcess.IO & Sendable {
     // NOP
     func resize(size: Terminal.Size) throws {}
 
+    func close() throws {
+        self.state.withLock {
+            if let stdin = $0.stdin {
+                stdin.close()
+                $0.stdin = nil
+            }
+
+            if let stdout = $0.stdout {
+                stdout.close()
+                $0.stdout = nil
+            }
+
+            if let stderr = $0.stderr {
+                stderr.close()
+                $0.stderr = nil
+            }
+        }
+    }
+
     func closeStdin() throws {
         self.state.withLock {
             if let stdin = $0.stdin {
@@ -129,12 +154,15 @@ final class StandardIO: ManagedProcess.IO & Sendable {
         try self.state.withLock {
             if let stdin = $0.stdinPipe {
                 try stdin.fileHandleForReading.close()
+                $0.stdinPipe = nil
             }
             if let stdout = $0.stdoutPipe {
                 try stdout.fileHandleForWriting.close()
+                $0.stdoutPipe = nil
             }
             if let stderr = $0.stderrPipe {
                 try stderr.fileHandleForWriting.close()
+                $0.stderrPipe = nil
             }
         }
     }

vminitd/Sources/vminitd/TerminalIO.swift

@@ -16,17 +16,20 @@
 
 import ContainerizationOS
 import Foundation
+import LCShim
 import Logging
 import Synchronization
 
 final class TerminalIO: ManagedProcess.IO & Sendable {
     private struct State {
+        var stdinSocket: Socket?
+        var stdoutSocket: Socket?
+
         var stdin: IOPair?
         var stdout: IOPair?
+        var parent: Terminal?
     }
 
-    private let parent: Terminal
-    private let child: Terminal
     private let log: Logger?
     private let hostStdio: HostStdio
     private let state: Mutex<State>
@@ -35,29 +38,24 @@ final class TerminalIO: ManagedProcess.IO & Sendable {
         stdio: HostStdio,
         log: Logger?
     ) throws {
-        let pair = try Terminal.create()
-        self.parent = pair.parent
-        self.child = pair.child
-        self.state = Mutex(State())
         self.hostStdio = stdio
         self.log = log
+        self.state = Mutex(State())
     }
 
     func resize(size: Terminal.Size) throws {
-        try parent.resize(size: size)
+        try self.state.withLock {
+            if let parent = $0.parent {
+                try parent.resize(size: size)
+            }
+        }
     }
 
     func start(process: inout Command) throws {
         try self.state.withLock {
-            let ptyHandle = self.child.handle
-            let useHandles = self.hostStdio.stdin != nil || self.hostStdio.stdout != nil
-            // We currently set stdin to the controlling terminal always, so
-            // it must be a valid pty descriptor.
-            process.stdin = useHandles ? ptyHandle : nil
-
-            let stdoutHandle = useHandles ? ptyHandle : nil
-            process.stdout = stdoutHandle
-            process.stderr = stdoutHandle
+            process.stdin = nil
+            process.stdout = nil
+            process.stderr = nil
 
             if let stdinPort = self.hostStdio.stdin {
                 let type = VsockType(
@@ -66,15 +64,7 @@ final class TerminalIO: ManagedProcess.IO & Sendable {
                 )
                 let stdinSocket = try Socket(type: type, closeOnDeinit: false)
                 try stdinSocket.connect()
-
-                let pair = IOPair(
-                    readFrom: stdinSocket,
-                    writeTo: self.parent.handle,
-                    logger: self.log
-                )
-                $0.stdin = pair
-
-                try pair.relay()
+                $0.stdinSocket = stdinSocket
             }
 
             if let stdoutPort = self.hostStdio.stdout {
@@ -84,26 +74,74 @@ final class TerminalIO: ManagedProcess.IO & Sendable {
                 )
                 let stdoutSocket = try Socket(type: type, closeOnDeinit: false)
                 try stdoutSocket.connect()
-
-                let pair = IOPair(
-                    readFrom: self.parent.handle,
-                    writeTo: stdoutSocket,
-                    logger: self.log
-                )
-                $0.stdout = pair
-
-                try pair.relay()
+                $0.stdoutSocket = stdoutSocket
             }
         }
     }
 
-    func closeStdin() throws {
-        self.state.withLock {
-            $0.stdin?.close()
+    func attach(pid: Int32, fd: Int32) throws {
+        try self.state.withLock {
+            let containerFd = CZ_pidfd_open(pid, 0)
+            guard containerFd != -1 else {
+                throw POSIXError.fromErrno()
+            }
+            defer { Foundation.close(Int32(containerFd)) }
+
+            let hostFd = CZ_pidfd_getfd(containerFd, fd, 0)
+            guard hostFd != -1 else {
+                throw POSIXError.fromErrno()
+            }
+
+            let term = try Terminal(descriptor: Int32(hostFd), setInitState: false)
+            $0.parent = term
+
+            if let stdinSocket = $0.stdinSocket {
+                let pair = IOPair(
+                    readFrom: stdinSocket,
+                    writeTo: term,
+                    reason: "TerminalIO stdin",
+                    logger: log
+                )
+                try pair.relay(ignoreHup: true)
+                $0.stdin = pair
+            }
+
+            if let stdoutSocket = $0.stdoutSocket {
+                let pair = IOPair(
+                    readFrom: term,
+                    writeTo: stdoutSocket,
+                    reason: "TerminalIO stdout",
+                    logger: log
+                )
+                try pair.relay(ignoreHup: true)
+                $0.stdout = pair
+            }
         }
     }
 
-    func closeAfterExec() throws {
-        try child.close()
+    func close() throws {
+        self.state.withLock {
+            if let stdin = $0.stdin {
+                stdin.close()
+                $0.stdin = nil
+            }
+            if let stdout = $0.stdout {
+                stdout.close()
+                $0.stdout = nil
+            }
+            $0.parent = nil
+        }
+    }
+
+    // NOP
+    func closeAfterExec() throws {}
+
+    func closeStdin() throws {
+        self.state.withLock {
+            if let stdin = $0.stdin {
+                stdin.close()
+                $0.stdin = nil
+            }
+        }
     }
 }