diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 27ead98b..1ca64634 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -215,6 +215,7 @@ struct IntegrationSuite: AsyncParsableCommand { "nested virt": testNestedVirtualizationEnabled, "container manager": testContainerManagerCreate, "container reuse": testContainerReuse, + "container /dev/console": testContainerDevConsole, ] var passed = 0 diff --git a/Sources/Integration/VMTests.swift b/Sources/Integration/VMTests.swift index 851e19ad..02d3d376 100644 --- a/Sources/Integration/VMTests.swift +++ b/Sources/Integration/VMTests.swift @@ -257,6 +257,54 @@ extension IntegrationSuite { } } + func testContainerDevConsole() async throws { + let id = "test-container-devconsole" + + let bs = try await bootstrap() + + let manager = try ContainerManager(vmm: bs.vmm) + defer { + try? manager.delete(id) + } + + let buffer = BufferWriter() + let container = try await manager.create( + id, + image: bs.image, + rootfs: bs.rootfs + ) { config in + // We mount devtmpfs by default, and while this includes creating + // /dev/console typically that'll be pointing to /dev/hvc0 (the + // virtio serial console). This is just a character device, so a trivial + // way to check that our bind mounted console setup worked is by just + // parsing `mount`'s output and looking for /dev/console as it wouldn't + // be there normally without our dance. + config.process.arguments = ["mount"] + config.process.terminal = true + config.process.stdout = buffer + } + + try await container.create() + try await container.start() + + let status = try await container.wait() + try await container.stop() + guard status == 0 else { + throw IntegrationError.assert(msg: "process status \(status) != 0") + } + + guard let str = String(data: buffer.data, encoding: .utf8) else { + throw IntegrationError.assert( + msg: "failed to convert standard output to a UTF8 string") + } + + let devConsole = "/dev/console" + guard str.contains(devConsole) else { + throw IntegrationError.assert( + msg: "process should have \(devConsole) in `mount` output") + } + } + private func createMountDirectory() throws -> URL { let dir = FileManager.default.uniqueTemporaryDirectory(create: true) try "hello".write(to: dir.appendingPathComponent("hi.txt"), atomically: true, encoding: .utf8) diff --git a/Sources/cctl/RunCommand.swift b/Sources/cctl/RunCommand.swift index 001132bc..40032480 100644 --- a/Sources/cctl/RunCommand.swift +++ b/Sources/cctl/RunCommand.swift @@ -68,7 +68,8 @@ extension Application { @Option(name: .long, help: "Current working directory") var cwd: String = "/" - @Argument var arguments: [String] = ["/bin/sh"] + @Argument(parsing: .captureForPassthrough) + var arguments: [String] = ["/bin/sh"] func run() async throws { let kernel = Kernel( diff --git a/vminitd/Sources/vmexec/Console.swift b/vminitd/Sources/vmexec/Console.swift new file mode 100644 index 00000000..e714638d --- /dev/null +++ b/vminitd/Sources/vmexec/Console.swift @@ -0,0 +1,62 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the Containerization project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation +import Musl + +class Console { + let master: Int32 + let slavePath: String + + init() throws { + let masterFD = open("/dev/ptmx", O_RDWR | O_NOCTTY | O_CLOEXEC) + guard masterFD != -1 else { + throw App.Errno(stage: "open_ptmx") + } + + guard unlockpt(masterFD) == 0 else { + throw App.Errno(stage: "unlockpt") + } + + guard let slavePath = ptsname(masterFD) else { + throw App.Errno(stage: "ptsname") + } + + self.master = masterFD + self.slavePath = String(cString: slavePath) + } + + func configureStdIO() throws { + let path = self.slavePath + let slaveFD = open(path, O_RDWR) + guard slaveFD != -1 else { + throw App.Errno(stage: "open_pts") + } + defer { Musl.close(slaveFD) } + + for fd: Int32 in 0...2 { + guard dup3(slaveFD, fd, 0) != -1 else { + throw App.Errno(stage: "dup3") + } + } + } + + func close() throws { + guard Musl.close(self.master) == 0 else { + throw App.Errno(stage: "close") + } + } +} diff --git a/vminitd/Sources/vmexec/ExecCommand.swift b/vminitd/Sources/vmexec/ExecCommand.swift index 545935c2..e26e4b86 100644 --- a/vminitd/Sources/vmexec/ExecCommand.swift +++ b/vminitd/Sources/vmexec/ExecCommand.swift @@ -62,37 +62,68 @@ struct ExecCommand: ParsableCommand { process: ContainerizationOCI.Process, log: Logger ) throws { - // CLOEXEC the pipe fd that signals process readiness. - let syncfd = FileHandle(fileDescriptor: 3) - if fcntl(3, F_SETFD, FD_CLOEXEC) == -1 { - throw App.Errno(stage: "cloexec(syncfd)") - } + let syncPipe = FileHandle(fileDescriptor: 3) + let ackPipe = FileHandle(fileDescriptor: 4) try Self.enterNS(path: "/proc/\(self.parentPid)/ns/cgroup", nsType: CLONE_NEWCGROUP) try Self.enterNS(path: "/proc/\(self.parentPid)/ns/pid", nsType: CLONE_NEWPID) try Self.enterNS(path: "/proc/\(self.parentPid)/ns/uts", nsType: CLONE_NEWUTS) try Self.enterNS(path: "/proc/\(self.parentPid)/ns/mnt", nsType: CLONE_NEWNS) - let childPipe = Pipe() - try childPipe.setCloexec() let processID = fork() guard processID != -1 else { - try? childPipe.fileHandleForReading.close() - try? childPipe.fileHandleForWriting.close() - try? syncfd.close() + try? syncPipe.close() + try? ackPipe.close() throw App.Errno(stage: "fork") } if processID == 0 { // child - try childPipe.fileHandleForReading.close() - try syncfd.close() + // Wait for the grandparent to tell us that they acked our pid. + guard let data = try ackPipe.read(upToCount: App.ackPid.count) else { + throw App.Failure(message: "read ack pipe") + } + guard let pidAckStr = String(data: data, encoding: .utf8) else { + throw App.Failure(message: "convert ack pipe data to string") + } + + guard pidAckStr == App.ackPid else { + throw App.Failure(message: "received invalid acknowledgement string: \(pidAckStr)") + } guard setsid() != -1 else { throw App.Errno(stage: "setsid()") } + if process.terminal { + let pty = try Console() + try pty.configureStdIO() + var masterFD = pty.master + + let data = Data(bytes: &masterFD, count: MemoryLayout.size(ofValue: masterFD)) + try syncPipe.write(contentsOf: data) + try syncPipe.close() + + // Wait for the grandparent to tell us that they acked our console. + guard let data = try ackPipe.read(upToCount: App.ackConsole.count) else { + throw App.Failure(message: "read ack pipe") + } + + guard let consoleAckStr = String(data: data, encoding: .utf8) else { + throw App.Failure(message: "convert ack pipe data to string") + } + + guard consoleAckStr == App.ackConsole else { + throw App.Failure(message: "received invalid acknowledgement string: \(consoleAckStr)") + } + + guard ioctl(0, UInt(TIOCSCTTY), 0) != -1 else { + throw App.Errno(stage: "setctty(0)") + } + try pty.close() + } + // Apply O_CLOEXEC to all file descriptors except stdio. // This ensures that all unwanted fds we may have accidentally // inherited are marked close-on-exec so they stay out of the @@ -106,26 +137,13 @@ struct ExecCommand: ParsableCommand { // Set uid, gid, and supplementary groups try App.setPermissions(user: process.user) - if process.terminal { - guard ioctl(0, UInt(TIOCSCTTY), 0) != -1 else { - throw App.Errno(stage: "setctty()") - } - } - try App.exec(process: process) } else { // parent process - try childPipe.fileHandleForWriting.close() - - // wait until the pipe is closed then carry on. - _ = try childPipe.fileHandleForReading.readToEnd() - try childPipe.fileHandleForReading.close() - - // send our child's pid to our parent before we exit. + // Send our child's pid to our parent before we exit. var childPid = processID let data = Data(bytes: &childPid, count: MemoryLayout.size(ofValue: childPid)) - try syncfd.write(contentsOf: data) - try syncfd.close() + try syncPipe.write(contentsOf: data) } } } diff --git a/vminitd/Sources/vmexec/Mount.swift b/vminitd/Sources/vmexec/Mount.swift index e3d52107..97c8140c 100644 --- a/vminitd/Sources/vmexec/Mount.swift +++ b/vminitd/Sources/vmexec/Mount.swift @@ -36,8 +36,7 @@ struct ContainerMount { } func configureConsole() throws { - let ptmx = self.rootfs.standardizingPath.appendingPathComponent("/dev/ptmx") - + let ptmx = self.rootfs.standardizingPath.appendingPathComponent("dev/ptmx") guard remove(ptmx) == 0 else { throw App.Errno(stage: "remove(ptmx)") } diff --git a/vminitd/Sources/vmexec/RunCommand.swift b/vminitd/Sources/vmexec/RunCommand.swift index c9316840..73913c08 100644 --- a/vminitd/Sources/vmexec/RunCommand.swift +++ b/vminitd/Sources/vmexec/RunCommand.swift @@ -49,92 +49,120 @@ struct RunCommand: ParsableCommand { try reOpenDevNull() } - private func execInNamespace(spec: ContainerizationOCI.Spec, log: Logger) throws { + private func childSetup( + spec: ContainerizationOCI.Spec, + ackPipe: FileHandle, + syncPipe: FileHandle, + log: Logger + ) throws { guard let process = spec.process else { - fatalError("no process configuration found in runtime spec") + throw App.Failure(message: "no process configuration found in runtime spec") } guard let root = spec.root else { - fatalError("no root found in runtime spec") + throw App.Failure(message: "no root found in runtime spec") } - let syncfd = FileHandle(fileDescriptor: 3) - if fcntl(3, F_SETFD, FD_CLOEXEC) == -1 { - throw App.Errno(stage: "cloexec(syncfd)") + // Wait for the grandparent to tell us that they acked our pid. + guard let data = try ackPipe.read(upToCount: App.ackPid.count) else { + throw App.Failure(message: "read ack pipe") } + guard let pidAckStr = String(data: data, encoding: .utf8) else { + throw App.Failure(message: "convert ack pipe data to string") + } + + guard pidAckStr == App.ackPid else { + throw App.Failure(message: "received invalid acknowledgement string: \(pidAckStr)") + } + + guard unshare(CLONE_NEWCGROUP) == 0 else { + throw App.Errno(stage: "unshare(cgroup)") + } + + guard setsid() != -1 else { + throw App.Errno(stage: "setsid()") + } + + try childRootSetup(rootfs: root, mounts: spec.mounts, log: log) + + if process.terminal { + let pty = try Console() + try pty.configureStdIO() + var masterFD = pty.master + + let data = Data(bytes: &masterFD, count: MemoryLayout.size(ofValue: masterFD)) + try syncPipe.write(contentsOf: data) + try syncPipe.close() + + // Wait for the grandparent to tell us that they acked our console. + guard let data = try ackPipe.read(upToCount: App.ackConsole.count) else { + throw App.Failure(message: "read ack pipe") + } + + guard let consoleAckStr = String(data: data, encoding: .utf8) else { + throw App.Failure(message: "convert ack pipe data to string") + } + + guard consoleAckStr == App.ackConsole else { + throw App.Failure(message: "received invalid acknowledgement string: \(consoleAckStr)") + } + + guard ioctl(0, UInt(TIOCSCTTY), 0) != -1 else { + throw App.Errno(stage: "setctty(0)") + } + + try mountConsole(path: pty.slavePath) + try pty.close() + } + + if !spec.hostname.isEmpty { + let errCode = spec.hostname.withCString { ptr in + Musl.sethostname(ptr, spec.hostname.count) + } + guard errCode == 0 else { + throw App.Errno(stage: "sethostname()") + } + } + + // Apply O_CLOEXEC to all file descriptors except stdio. + // This ensures that all unwanted fds we may have accidentally + // inherited are marked close-on-exec so they stay out of the + // container. + try App.applyCloseExecOnFDs() + + try App.setRLimits(rlimits: process.rlimits) + + // Change stdio to be owned by the requested user. + try App.fixStdioPerms(user: process.user) + + // Set uid, gid, and supplementary groups. + try App.setPermissions(user: process.user) + + // Finally execve the container process. + try App.exec(process: process) + } + + private func execInNamespace(spec: ContainerizationOCI.Spec, log: Logger) throws { + let syncPipe = FileHandle(fileDescriptor: 3) + let ackPipe = FileHandle(fileDescriptor: 4) guard unshare(CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWUTS) == 0 else { throw App.Errno(stage: "unshare(pid|mnt|uts)") } - let childPipe = Pipe() - try childPipe.setCloexec() let processID = fork() - guard processID != -1 else { - try? childPipe.fileHandleForReading.close() - try? childPipe.fileHandleForWriting.close() - try? syncfd.close() - + try? syncPipe.close() + try? ackPipe.close() throw App.Errno(stage: "fork") } if processID == 0 { // child - try childPipe.fileHandleForReading.close() - try syncfd.close() - - guard unshare(CLONE_NEWCGROUP) == 0 else { - throw App.Errno(stage: "unshare(cgroup)") - } - - guard setsid() != -1 else { - throw App.Errno(stage: "setsid()") - } - - try childRootSetup(rootfs: root, mounts: spec.mounts, log: log) - - if !spec.hostname.isEmpty { - let errCode = spec.hostname.withCString { ptr in - Musl.sethostname(ptr, spec.hostname.count) - } - guard errCode == 0 else { - throw App.Errno(stage: "sethostname()") - } - } - - // Apply O_CLOEXEC to all file descriptors except stdio. - // This ensures that all unwanted fds we may have accidentally - // inherited are marked close-on-exec so they stay out of the - // container. - try App.applyCloseExecOnFDs() - - try App.setRLimits(rlimits: process.rlimits) - - // Change stdio to be owned by the requested user. - try App.fixStdioPerms(user: process.user) - - // Set uid, gid, and supplementary groups. - try App.setPermissions(user: process.user) - - if process.terminal { - guard ioctl(0, UInt(TIOCSCTTY), 0) != -1 else { - throw App.Errno(stage: "setctty()") - } - } - - try App.exec(process: process) + try childSetup(spec: spec, ackPipe: ackPipe, syncPipe: syncPipe, log: log) } else { // parent process - try childPipe.fileHandleForWriting.close() - - // wait until the pipe is closed then carry on. - _ = try childPipe.fileHandleForReading.readToEnd() - try childPipe.fileHandleForReading.close() - - // send our child's pid to our parent before we exit. + // Send our child's pid before we exit. var childPid = processID let data = Data(bytes: &childPid, count: MemoryLayout.size(ofValue: childPid)) - - try syncfd.write(contentsOf: data) - try syncfd.close() + try syncPipe.write(contentsOf: data) } } @@ -222,7 +250,6 @@ struct RunCommand: ParsableCommand { if newRoot <= 0 { throw App.Errno(stage: "open(newroot)") } - defer { close(newRoot) } // change cwd to the new root @@ -257,4 +284,19 @@ struct RunCommand: ParsableCommand { _ = cStringCopy.initialize(from: cString) return UnsafeMutablePointer(cStringCopy.baseAddress) } + + private func mountConsole(path: String) throws { + let console = "/dev/console" + if access(console, F_OK) != 0 { + let fd = open(console, O_RDWR | O_CREAT, mode_t(UInt16(0o600))) + guard fd != -1 else { + throw App.Errno(stage: "open(/dev/console)") + } + close(fd) + } + + guard mount(path, console, "bind", UInt(MS_BIND), nil) == 0 else { + throw App.Errno(stage: "mount(console)") + } + } } diff --git a/vminitd/Sources/vmexec/vmexec.swift b/vminitd/Sources/vmexec/vmexec.swift index ab383ace..540c69e7 100644 --- a/vminitd/Sources/vmexec/vmexec.swift +++ b/vminitd/Sources/vmexec/vmexec.swift @@ -29,6 +29,9 @@ import Musl @main struct App: ParsableCommand { + static let ackPid = "AckPid" + static let ackConsole = "AckConsole" + static let configuration = CommandConfiguration( commandName: "vmexec", version: "0.1.0", @@ -161,4 +164,11 @@ extension App { let posix = POSIXError(.init(rawValue: errno)!, userInfo: ["stage": stage]) return ContainerizationError(.internalError, message: "\(info) \(String(describing: posix))") } + + static func Failure(message: String) -> ContainerizationError { + ContainerizationError( + .internalError, + message: message + ) + } } diff --git a/vminitd/Sources/vminitd/IOPair.swift b/vminitd/Sources/vminitd/IOPair.swift index 0c277782..64de55ad 100644 --- a/vminitd/Sources/vminitd/IOPair.swift +++ b/vminitd/Sources/vminitd/IOPair.swift @@ -21,104 +21,162 @@ import Logging import Synchronization final class IOPair: Sendable { - let readFrom: IOCloser - let writeTo: IOCloser - nonisolated(unsafe) let buffer: UnsafeMutableBufferPointer + private let io: Mutex private let logger: Logger? + private let reason: String - private let done: Atomic + private struct IO { + let from: IOCloser + let to: IOCloser + let buffer: UnsafeMutableBufferPointer + var closed: Bool + let logger: Logger? - init(readFrom: IOCloser, writeTo: IOCloser, logger: Logger? = nil) { - self.readFrom = readFrom - self.writeTo = writeTo - self.done = Atomic(false) - self.buffer = UnsafeMutableBufferPointer.allocate(capacity: Int(getpagesize())) - self.logger = logger - } + func drain() { + let readFrom = OSFile(fd: from.fileDescriptor) + let writeTo = OSFile(fd: to.fileDescriptor) - func relay() throws { - let readFromFd = self.readFrom.fileDescriptor - let writeToFd = self.writeTo.fileDescriptor - - let readFrom = OSFile(fd: readFromFd) - let writeTo = OSFile(fd: writeToFd) - - try ProcessSupervisor.default.poller.add(readFromFd, mask: EPOLLIN) { mask in - if mask.isHangup && !mask.readyToRead { - self.close() - return - } - // Loop so that in the case that someone wrote > buf.count down the pipe - // we properly will drain it fully. while true { - let r = readFrom.read(self.buffer) + let r = readFrom.read(buffer) if r.read > 0 { let view = UnsafeMutableBufferPointer( - start: self.buffer.baseAddress, + start: buffer.baseAddress, count: r.read ) let w = writeTo.write(view) if w.wrote != r.read { - self.logger?.error("stopping relay: short write for stdio") - self.close() return } } switch r.action { - case .error(let errno): - self.logger?.error("failed with errno \(errno) while reading for fd \(readFromFd)") - fallthrough - case .eof: - self.close() - self.logger?.debug("closing relay for \(readFromFd)") - return - case .again: - // We read all we could, exit. - if mask.isHangup { - self.close() - } + case .eof, .again, .error(_): return default: break } } } + + mutating func close() { + if self.closed { + return + } + + // Try and drain IO first. + self.drain() + + // Remove the fd from our global epoll instance first. + let readFromFd = self.from.fileDescriptor + do { + try ProcessSupervisor.default.poller.delete(readFromFd) + } catch { + self.logger?.error("failed to delete fd from epoll \(readFromFd): \(error)") + } + + do { + try self.from.close() + } catch { + self.logger?.error("failed to close reader fd for IOPair: \(error)") + } + + do { + try self.to.close() + } catch { + self.logger?.error("failed to close writer fd for IOPair: \(error)") + } + self.buffer.deallocate() + self.closed = true + } + } + + init( + readFrom: IOCloser, + writeTo: IOCloser, + reason: String, + logger: Logger? = nil + ) { + let buffer = UnsafeMutableBufferPointer.allocate(capacity: Int(getpagesize())) + self.io = Mutex( + IO( + from: readFrom, + to: writeTo, + buffer: buffer, + closed: false, + logger: logger + )) + self.reason = reason + self.logger = logger + } + + func relay(ignoreHup: Bool = false) throws { + self.logger?.info("setting up relay for \(reason)") + + let (readFromFd, writeToFd) = self.io.withLock { io in + (io.from.fileDescriptor, io.to.fileDescriptor) + } + + let readFrom = OSFile(fd: readFromFd) + let writeTo = OSFile(fd: writeToFd) + + try ProcessSupervisor.default.poller.add(readFromFd, mask: EPOLLIN) { mask in + self.io.withLock { io in + if io.closed { + return + } + + if mask.isHangup && !mask.readyToRead { + self.logger?.debug("received EPOLLHUP with no EPOLLIN") + if !ignoreHup { + io.close() + } + return + } + + // Loop so we drain fully. + while true { + let r = readFrom.read(io.buffer) + if r.read > 0 { + let view = UnsafeMutableBufferPointer( + start: io.buffer.baseAddress, + count: r.read + ) + + let w = writeTo.write(view) + if w.wrote != r.read { + self.logger?.error("stopping relay: short write for stdio") + io.close() + return + } + } + + switch r.action { + case .error(let errno): + self.logger?.error("failed with errno \(errno) while reading for fd \(readFromFd)") + fallthrough + case .eof: + self.logger?.debug("closing relay for \(readFromFd)") + io.close() + return + case .again: + if mask.isHangup && !ignoreHup { + self.logger?.error("received EPOLLHUP and EAGAIN exiting") + self.close() + } + return + default: + break + } + } + } + } } func close() { - guard - self.done.compareExchange( - expected: false, - desired: true, - successOrdering: .acquiringAndReleasing, - failureOrdering: .acquiring - ).exchanged - else { - return - } - - self.buffer.deallocate() - - let readFromFd = self.readFrom.fileDescriptor - // Remove the fd from our global epoll instance first. - do { - try ProcessSupervisor.default.poller.delete(readFromFd) - } catch { - self.logger?.error("failed to delete fd from epoll \(readFromFd): \(error)") - } - - do { - try self.readFrom.close() - } catch { - self.logger?.error("failed to close reader fd for IOPair: \(error)") - } - - do { - try self.writeTo.close() - } catch { - self.logger?.error("failed to close writer fd for IOPair: \(error)") + self.io.withLock { io in + self.logger?.info("closing relay for \(reason)") + io.close() } } } diff --git a/vminitd/Sources/vminitd/ManagedProcess.swift b/vminitd/Sources/vminitd/ManagedProcess.swift index 4cfff235..05734bed 100644 --- a/vminitd/Sources/vminitd/ManagedProcess.swift +++ b/vminitd/Sources/vminitd/ManagedProcess.swift @@ -29,8 +29,11 @@ final class ManagedProcess: Sendable { private let log: Logger private let process: Command private let state: Mutex - private let syncfd: Pipe private let owningPid: Int32? + private let ackPipe: FileHandle + private let syncPipe: FileHandle + private let terminal: Bool + private let bundle: ContainerizationOCI.Bundle private struct State { init(io: IO) { @@ -51,10 +54,12 @@ final class ManagedProcess: Sendable { // swiftlint: disable type_name protocol IO { + func attach(pid: Int32, fd: Int32) throws func start(process: inout Command) throws - func closeAfterExec() throws func resize(size: Terminal.Size) throws + func close() throws func closeStdin() throws + func closeAfterExec() throws } // swiftlint: enable type_name @@ -62,6 +67,9 @@ final class ManagedProcess: Sendable { log[metadataKey: "id"] = "\(id)" } + private static let ackPid = "AckPid" + private static let ackConsole = "AckConsole" + init( id: String, stdio: HostStdio, @@ -75,9 +83,13 @@ final class ManagedProcess: Sendable { self.log = log self.owningPid = owningPid - let syncfd = Pipe() - try syncfd.setCloexec() - self.syncfd = syncfd + let syncPipe = Pipe() + try syncPipe.setCloexec() + self.syncPipe = syncPipe.fileHandleForReading + + let ackPipe = Pipe() + try ackPipe.setCloexec() + self.ackPipe = ackPipe.fileHandleForWriting let args: [String] if let owningPid { @@ -95,7 +107,10 @@ final class ManagedProcess: Sendable { var process = Command( "/sbin/vmexec", arguments: args, - extraFiles: [syncfd.fileHandleForWriting] + extraFiles: [ + syncPipe.fileHandleForWriting, + ackPipe.fileHandleForReading, + ] ) var io: IO @@ -121,6 +136,8 @@ final class ManagedProcess: Sendable { try io.start(process: &process) self.process = process + self.terminal = stdio.terminal + self.bundle = bundle self.state = Mutex(State(io: io)) } } @@ -136,30 +153,77 @@ extension ManagedProcess { // Start the underlying process. try process.start() + defer { + try? self.ackPipe.close() + try? self.syncPipe.close() + } // Close our side of any pipes. - try syncfd.fileHandleForWriting.close() try $0.io.closeAfterExec() - guard let piddata = try syncfd.fileHandleForReading.readToEnd() else { + let size = MemoryLayout.size + guard let piddata = try syncPipe.read(upToCount: size) else { throw ContainerizationError(.internalError, message: "no pid data from sync pipe") } - let i = piddata.withUnsafeBytes { ptr in + guard piddata.count == size else { + throw ContainerizationError(.internalError, message: "invalid payload") + } + + let pid = piddata.withUnsafeBytes { ptr in ptr.load(as: Int32.self) } - log.info("got back pid data \(i)") - $0.pid = i + log.info( + "got back pid data", + metadata: [ + "id": "\(pid)" + ]) + $0.pid = pid + + // Ack the pid from the child. + log.info( + "sending pid acknowledgement", + metadata: [ + "pid": "\(pid)" + ]) + try self.ackPipe.write(contentsOf: Self.ackPid.data(using: .utf8)!) + + if self.terminal { + log.info( + "wait for pty fd", + metadata: [ + "id": "\(id)" + ]) + + // Wait for a new write that will contain the pty fd if we asked for one. + guard let ptyFd = try syncPipe.read(upToCount: size) else { + throw ContainerizationError( + .internalError, + message: "no pty data from sync pipe" + ) + } + let fd = ptyFd.withUnsafeBytes { ptr in + ptr.load(as: Int32.self) + } + log.info( + "received pty fd from container, attaching", + metadata: [ + "id": "\(id)" + ]) + + try $0.io.attach(pid: pid, fd: fd) + try self.ackPipe.write(contentsOf: Self.ackConsole.data(using: .utf8)!) + } log.info( "started managed process", metadata: [ - "pid": "\(i)", + "pid": "\(pid)", "id": "\(id)", ]) - return i + return pid } } @@ -173,6 +237,12 @@ extension ManagedProcess { $0.exitStatus = status + do { + try $0.io.close() + } catch { + self.log.error("failed to close io for process: \(error)") + } + for waiter in $0.waiters { waiter.resume(returning: status) } diff --git a/vminitd/Sources/vminitd/StandardIO.swift b/vminitd/Sources/vminitd/StandardIO.swift index dd1f908d..13c310f9 100644 --- a/vminitd/Sources/vminitd/StandardIO.swift +++ b/vminitd/Sources/vminitd/StandardIO.swift @@ -43,6 +43,9 @@ final class StandardIO: ManagedProcess.IO & Sendable { self.state = Mutex(State()) } + // NOP + func attach(pid: Int32, fd: Int32) throws {} + func start(process: inout Command) throws { try self.state.withLock { if let stdinPort = self.hostStdio.stdin { @@ -60,6 +63,7 @@ final class StandardIO: ManagedProcess.IO & Sendable { let pair = IOPair( readFrom: stdinSocket, writeTo: inPipe.fileHandleForWriting, + reason: "StandardIO stdin", logger: log ) $0.stdin = pair @@ -82,6 +86,7 @@ final class StandardIO: ManagedProcess.IO & Sendable { let pair = IOPair( readFrom: outPipe.fileHandleForReading, writeTo: stdoutSocket, + reason: "StandardIO stdout", logger: log ) $0.stdout = pair @@ -104,6 +109,7 @@ final class StandardIO: ManagedProcess.IO & Sendable { let pair = IOPair( readFrom: errPipe.fileHandleForReading, writeTo: stderrSocket, + reason: "StandardIO stderr", logger: log ) $0.stderr = pair @@ -116,6 +122,25 @@ final class StandardIO: ManagedProcess.IO & Sendable { // NOP func resize(size: Terminal.Size) throws {} + func close() throws { + self.state.withLock { + if let stdin = $0.stdin { + stdin.close() + $0.stdin = nil + } + + if let stdout = $0.stdout { + stdout.close() + $0.stdout = nil + } + + if let stderr = $0.stderr { + stderr.close() + $0.stderr = nil + } + } + } + func closeStdin() throws { self.state.withLock { if let stdin = $0.stdin { @@ -129,12 +154,15 @@ final class StandardIO: ManagedProcess.IO & Sendable { try self.state.withLock { if let stdin = $0.stdinPipe { try stdin.fileHandleForReading.close() + $0.stdinPipe = nil } if let stdout = $0.stdoutPipe { try stdout.fileHandleForWriting.close() + $0.stdoutPipe = nil } if let stderr = $0.stderrPipe { try stderr.fileHandleForWriting.close() + $0.stderrPipe = nil } } } diff --git a/vminitd/Sources/vminitd/TerminalIO.swift b/vminitd/Sources/vminitd/TerminalIO.swift index c8815848..4e64f717 100644 --- a/vminitd/Sources/vminitd/TerminalIO.swift +++ b/vminitd/Sources/vminitd/TerminalIO.swift @@ -16,17 +16,20 @@ import ContainerizationOS import Foundation +import LCShim import Logging import Synchronization final class TerminalIO: ManagedProcess.IO & Sendable { private struct State { + var stdinSocket: Socket? + var stdoutSocket: Socket? + var stdin: IOPair? var stdout: IOPair? + var parent: Terminal? } - private let parent: Terminal - private let child: Terminal private let log: Logger? private let hostStdio: HostStdio private let state: Mutex @@ -35,29 +38,24 @@ final class TerminalIO: ManagedProcess.IO & Sendable { stdio: HostStdio, log: Logger? ) throws { - let pair = try Terminal.create() - self.parent = pair.parent - self.child = pair.child - self.state = Mutex(State()) self.hostStdio = stdio self.log = log + self.state = Mutex(State()) } func resize(size: Terminal.Size) throws { - try parent.resize(size: size) + try self.state.withLock { + if let parent = $0.parent { + try parent.resize(size: size) + } + } } func start(process: inout Command) throws { try self.state.withLock { - let ptyHandle = self.child.handle - let useHandles = self.hostStdio.stdin != nil || self.hostStdio.stdout != nil - // We currently set stdin to the controlling terminal always, so - // it must be a valid pty descriptor. - process.stdin = useHandles ? ptyHandle : nil - - let stdoutHandle = useHandles ? ptyHandle : nil - process.stdout = stdoutHandle - process.stderr = stdoutHandle + process.stdin = nil + process.stdout = nil + process.stderr = nil if let stdinPort = self.hostStdio.stdin { let type = VsockType( @@ -66,15 +64,7 @@ final class TerminalIO: ManagedProcess.IO & Sendable { ) let stdinSocket = try Socket(type: type, closeOnDeinit: false) try stdinSocket.connect() - - let pair = IOPair( - readFrom: stdinSocket, - writeTo: self.parent.handle, - logger: self.log - ) - $0.stdin = pair - - try pair.relay() + $0.stdinSocket = stdinSocket } if let stdoutPort = self.hostStdio.stdout { @@ -84,26 +74,74 @@ final class TerminalIO: ManagedProcess.IO & Sendable { ) let stdoutSocket = try Socket(type: type, closeOnDeinit: false) try stdoutSocket.connect() - - let pair = IOPair( - readFrom: self.parent.handle, - writeTo: stdoutSocket, - logger: self.log - ) - $0.stdout = pair - - try pair.relay() + $0.stdoutSocket = stdoutSocket } } } - func closeStdin() throws { - self.state.withLock { - $0.stdin?.close() + func attach(pid: Int32, fd: Int32) throws { + try self.state.withLock { + let containerFd = CZ_pidfd_open(pid, 0) + guard containerFd != -1 else { + throw POSIXError.fromErrno() + } + defer { Foundation.close(Int32(containerFd)) } + + let hostFd = CZ_pidfd_getfd(containerFd, fd, 0) + guard hostFd != -1 else { + throw POSIXError.fromErrno() + } + + let term = try Terminal(descriptor: Int32(hostFd), setInitState: false) + $0.parent = term + + if let stdinSocket = $0.stdinSocket { + let pair = IOPair( + readFrom: stdinSocket, + writeTo: term, + reason: "TerminalIO stdin", + logger: log + ) + try pair.relay(ignoreHup: true) + $0.stdin = pair + } + + if let stdoutSocket = $0.stdoutSocket { + let pair = IOPair( + readFrom: term, + writeTo: stdoutSocket, + reason: "TerminalIO stdout", + logger: log + ) + try pair.relay(ignoreHup: true) + $0.stdout = pair + } } } - func closeAfterExec() throws { - try child.close() + func close() throws { + self.state.withLock { + if let stdin = $0.stdin { + stdin.close() + $0.stdin = nil + } + if let stdout = $0.stdout { + stdout.close() + $0.stdout = nil + } + $0.parent = nil + } + } + + // NOP + func closeAfterExec() throws {} + + func closeStdin() throws { + self.state.withLock { + if let stdin = $0.stdin { + stdin.close() + $0.stdin = nil + } + } } }