CI observability enhancements. (#1193)

- Adds a a `--log-root` option to `swift system start`, propagating the
value as `CONTAINER_LOG_ROOT` to services for logging to files instead
of the OS log facility. This is not a "production" capability as it
neither merges nor rotates logs.
- Currently we don't collect logs on CI builds, and we don't have
permission to run the `log` command there. The PR adds `--log-root` to
the CI test phase, archives the results, and uploads the archive as an
artifact.
- Use FilePath from swift-system for the log root. Foundation URL is a
bit of a footgun for filesystem paths, so unless we identify a
showstopper, we should incrementally transition to this type everywhere
except where we really need network URLs.
- Output the hostname of the CI runner at the start of the test phase so
we can identify runner-specific issues where they exist.
- Fix formatting for log messages with multiple metadata items, and fix
unstructured messages on instances that weren't found using `grep -r
'log\.' Sources`.
- Adds command reference documentation for `--log-root`.
This commit is contained in:
J Logan
2026-02-19 17:47:41 -08:00
committed by GitHub
parent 3425f04206
commit d29947121f
31 changed files with 628 additions and 117 deletions
@@ -17,6 +17,7 @@
import ContainerXPC
import ContainerizationError
import Foundation
import SystemPackage
public enum ClientHealthCheck {
static let serviceIdentifier = "com.apple.container.apiserver"
@@ -37,6 +38,7 @@ extension ClientHealthCheck {
guard let installRootValue = reply.string(key: .installRoot), let installRoot = URL(string: installRootValue) else {
throw ContainerizationError(.internalError, message: "failed to decode installRoot in health check")
}
let logRoot = reply.string(key: .logRoot).map { FilePath($0) }
guard let apiServerVersion = reply.string(key: .apiServerVersion) else {
throw ContainerizationError(.internalError, message: "failed to decode apiServerVersion in health check")
}
@@ -52,6 +54,7 @@ extension ClientHealthCheck {
return .init(
appRoot: appRoot,
installRoot: installRoot,
logRoot: logRoot,
apiServerVersion: apiServerVersion,
apiServerCommit: apiServerCommit,
apiServerBuild: apiServerBuild,
@@ -15,6 +15,7 @@
//===----------------------------------------------------------------------===//
import Foundation
import SystemPackage
/// Snapshot of the health of container services and resources
public struct SystemHealth: Sendable, Codable {
@@ -24,6 +25,9 @@ public struct SystemHealth: Sendable, Codable {
/// The full pathname of the application install root.
public let installRoot: URL
/// The full pathname of the application install root.
public let logRoot: FilePath?
/// The release version of the container services.
public let apiServerVersion: String
@@ -61,6 +61,7 @@ public enum XPCKeys: String {
case ping
case appRoot
case installRoot
case logRoot
case apiServerVersion
case apiServerCommit
case apiServerBuild
@@ -119,7 +119,12 @@ public actor ContainersService {
}
} catch {
try? FileManager.default.removeItem(at: dir)
log.warning("failed to load container", metadata: ["path": "\(dir.path)", "error": "\(error)"])
log.warning(
"failed to load container",
metadata: [
"path": "\(dir.path)",
"error": "\(error)",
])
}
}
return results
@@ -546,7 +551,12 @@ public actor ContainersService {
let waitFunc: ExitMonitor.WaitHandler = {
log.info("registering container with exit monitor")
let code = try await client.wait(id)
log.info("container finished in exit monitor", metadata: ["id": "\(id)", "rc": "\(code)"])
log.info(
"container finished in exit monitor",
metadata: [
"id": "\(id)",
"rc": "\(code)",
])
return code
}
@@ -882,7 +892,12 @@ public actor ContainersService {
private func handleContainerExit(id: String, code: ExitStatus?, context: AsyncLock.Context) async throws {
if let code {
self.log.info("handling container exit", metadata: ["id": "\(id)", "rc": "\(code)"])
self.log.info(
"handling container exit",
metadata: [
"id": "\(id)",
"rc": "\(code)",
])
}
var state: ContainerState
@@ -916,7 +931,12 @@ public actor ContainersService {
do {
try await client.shutdown()
} catch {
self.log.error("failed to shutdown sandbox service", metadata: ["id": "\(id)", "error": "\(error)"])
self.log.error(
"failed to shutdown sandbox service",
metadata: [
"id": "\(id)",
"error": "\(error)",
])
}
}
@@ -927,7 +947,12 @@ public actor ContainersService {
try ServiceManager.deregister(fullServiceLabel: label)
self.log.info("deregistered sandbox service", metadata: ["id": "\(id)"])
} catch {
self.log.error("failed to deregister sandbox service", metadata: ["id": "\(id)", "error": "\(error)"])
self.log.error(
"failed to deregister sandbox service",
metadata: [
"id": "\(id)",
"error": "\(error)",
])
}
// Best effort deallocate network attachments for the container. Don't throw on
@@ -999,7 +1024,12 @@ public actor ContainersService {
do {
config = try bundle.configuration
} catch {
self.log.warning("failed to read bundle configuration during cleanup for container", metadata: ["id": "\(id)", "error": "\(error)"])
self.log.warning(
"failed to read bundle configuration during cleanup for container",
metadata: [
"id": "\(id)",
"error": "\(error)",
])
}
// Only try to deregister service if we have a valid config
@@ -1017,7 +1047,12 @@ public actor ContainersService {
do {
try bundle.delete()
} catch {
self.log.warning("failed to delete bundle for container", metadata: ["id": "\(id)", "error": "\(error)"])
self.log.warning(
"failed to delete bundle for container",
metadata: [
"id": "\(id)",
"error": "\(error)",
])
}
self.containers.removeValue(forKey: id)
@@ -21,15 +21,18 @@ import ContainerXPC
import Containerization
import Foundation
import Logging
import SystemPackage
public actor HealthCheckHarness {
private let appRoot: URL
private let installRoot: URL
private let logRoot: FilePath?
private let log: Logger
public init(appRoot: URL, installRoot: URL, log: Logger) {
public init(appRoot: URL, installRoot: URL, logRoot: FilePath?, log: Logger) {
self.appRoot = appRoot
self.installRoot = installRoot
self.logRoot = logRoot
self.log = log
}
@@ -38,6 +41,9 @@ public actor HealthCheckHarness {
let reply = message.reply()
reply.set(key: .appRoot, value: appRoot.absoluteString)
reply.set(key: .installRoot, value: installRoot.absoluteString)
if let logRoot {
reply.set(key: .logRoot, value: logRoot.string)
}
reply.set(key: .apiServerVersion, value: ReleaseVersion.singleLine(appName: "container-apiserver"))
reply.set(key: .apiServerCommit, value: get_git_commit().map { String(cString: $0) } ?? "unspecified")
// Extra optional fields for richer client display
@@ -337,7 +337,13 @@ public actor VolumesService {
try await store.create(volume)
log.info("created volume", metadata: ["name": "\(name)", "driver": "\(driver)", "isAnonymous": "\(volume.isAnonymous)"])
log.info(
"created volume",
metadata: [
"name": "\(name)",
"driver": "\(driver)",
"isAnonymous": "\(volume.isAnonymous)",
])
return volume
}