CI observability enhancements. (#1193)

- Adds a a `--log-root` option to `swift system start`, propagating the
value as `CONTAINER_LOG_ROOT` to services for logging to files instead
of the OS log facility. This is not a "production" capability as it
neither merges nor rotates logs.
- Currently we don't collect logs on CI builds, and we don't have
permission to run the `log` command there. The PR adds `--log-root` to
the CI test phase, archives the results, and uploads the archive as an
artifact.
- Use FilePath from swift-system for the log root. Foundation URL is a
bit of a footgun for filesystem paths, so unless we identify a
showstopper, we should incrementally transition to this type everywhere
except where we really need network URLs.
- Output the hostname of the CI runner at the start of the test phase so
we can identify runner-specific issues where they exist.
- Fix formatting for log messages with multiple metadata items, and fix
unstructured messages on instances that weren't found using `grep -r
'log\.' Sources`.
- Adds command reference documentation for `--log-root`.

Sources/ContainerCommands/Application.swift

@@ -172,6 +172,7 @@ public struct Application: AsyncLoggableCommand {
         return try PluginLoader(
             appRoot: systemHealth.appRoot,
             installRoot: systemHealth.installRoot,
+            logRoot: systemHealth.logRoot,
             pluginDirectories: pluginDirectories,
             pluginFactories: pluginFactories,
             log: bootstrapLogger

Sources/ContainerCommands/Image/ImageInspect.swift

@@ -63,9 +63,13 @@ extension Application {
             }
 
             if !allErrors.isEmpty {
-                let logger = Logger(label: "ImageInspect", factory: { _ in StderrLogHandler() })
                 for (name, error) in allErrors {
-                    logger.error("\(name): \(error.localizedDescription)")
+                    log.error(
+                        "image inspect failed",
+                        metadata: [
+                            "name": "\(name)",
+                            "error": "\(error.localizedDescription)",
+                        ])
                 }
 
                 throw InspectError(succeeded: succeededImages, failed: allErrors)

Sources/ContainerCommands/Network/NetworkDelete.swift

@@ -89,7 +89,7 @@ extension Application {
             }
 
             var failed = [String]()
-            let logger = log
+            let _log = log
             try await withThrowingTaskGroup(of: NetworkState?.self) { group in
                 for network in networks {
                     group.addTask {
@@ -100,7 +100,12 @@ extension Application {
                             print(network.id)
                             return nil
                         } catch {
-                            logger.error("failed to delete network \(network.id): \(error)")
+                            _log.error(
+                                "failed to delete network",
+                                metadata: [
+                                    "id": "\(network.id)",
+                                    "error": "\(error)",
+                                ])
                             return network
                         }
                     }

Sources/ContainerCommands/Network/NetworkPrune.swift

@@ -55,7 +55,12 @@ extension Application.NetworkCommand {
                     // Note: This failure may occur due to a race condition between the network/
                     // container collection above and a container run command that attaches to a
                     // network listed in the networksToPrune collection.
-                    log.error("failed to prune network", metadata: ["id": "\(network.id)", "error": "\(error)"])
+                    log.error(
+                        "failed to prune network",
+                        metadata: [
+                            "id": "\(network.id)",
+                            "error": "\(error)",
+                        ])
                 }
             }
 

Sources/ContainerCommands/System/SystemStart.swift

@@ -21,6 +21,7 @@ import ContainerPlugin
 import ContainerXPC
 import ContainerizationError
 import Foundation
+import SystemPackage
 import TerminalProgress
 
 extension Application {
@@ -42,6 +43,12 @@ extension Application {
             transform: { URL(filePath: $0) })
         var installRoot = InstallRoot.defaultURL
 
+        @Option(
+            name: .long,
+            help: "Path to the root directory for log data, using macOS log facility if not set",
+            transform: { FilePath($0) })
+        var logRoot: FilePath? = nil
+
         @Flag(
             name: .long,
             inversion: .prefixedEnableDisable,
@@ -85,7 +92,12 @@ extension Application {
             var env = PluginLoader.filterEnvironment()
             env[ApplicationRoot.environmentName] = appRoot.path(percentEncoded: false)
             env[InstallRoot.environmentName] = installRoot.path(percentEncoded: false)
-
+            if let logRoot {
+                env[LogRoot.environmentName] =
+                    logRoot.isAbsolute
+                    ? logRoot.string
+                    : FilePath(FileManager.default.currentDirectoryPath).appending(logRoot.components).string
+            }
             let plist = LaunchPlist(
                 label: "com.apple.container.apiserver",
                 arguments: args,

Sources/ContainerCommands/Volume/VolumeDelete.swift

@@ -68,7 +68,7 @@ extension Application.VolumeCommand {
             }
 
             var failed = [String]()
-            let logger = log
+            let _log = log
             try await withThrowingTaskGroup(of: Volume?.self) { group in
                 for volume in volumes {
                     group.addTask {
@@ -77,7 +77,12 @@ extension Application.VolumeCommand {
                             print(volume.id)
                             return nil
                         } catch {
-                            logger.error("failed to delete volume \(volume.id): \(error)")
+                            _log.error(
+                                "failed to delete volume",
+                                metadata: [
+                                    "id": "\(volume.id)",
+                                    "error": "\(error)",
+                                ])
                             return volume
                         }
                     }