From 8adb2155228c48a36dfbbad4a7351ead5527a79f Mon Sep 17 00:00:00 2001 From: Kathryn Baldauf Date: Tue, 5 Aug 2025 14:34:34 -0700 Subject: [PATCH] Native Builder: Add parser support for CMD and LABEL instructions (#448) This PR adds support for CMD and LABEL instructions in the native builder's parser. This also changes how options are tokenized. Options now include the raw string so that when constructing a command for instructions like CMD and RUN, we can use the exact user input without having to add logic in the tokenizer to know when we're parsing a command verses other options, etc. Closes https://github.com/apple/container/issues/428 and https://github.com/apple/container/issues/429 Signed-off-by: Kathryn Baldauf --- .../ContainerBuildIR/Graph/GraphBuilder.swift | 22 ++- .../Docker/DockerInstructionVisitor.swift | 12 +- .../Docker/DockerfileParser.swift | 105 ++++++++--- .../Docker/DockerfileTokenizer.swift | 5 +- .../Docker/Instructions/CopyInstruction.swift | 2 +- .../Instructions/DockerInstruction.swift | 22 ++- .../Docker/Instructions/RunInstruction.swift | 11 +- .../ContainerBuildParser/Types.swift | 14 +- .../ParserTests.swift | 133 +++++++++++++- .../TokenizerTests.swift | 163 +++++++++++++----- .../VisitorTests.swift | 80 +++++++-- 11 files changed, 477 insertions(+), 92 deletions(-) diff --git a/Sources/NativeBuilder/ContainerBuildIR/Graph/GraphBuilder.swift b/Sources/NativeBuilder/ContainerBuildIR/Graph/GraphBuilder.swift index a118c05c..f52d76a7 100644 --- a/Sources/NativeBuilder/ContainerBuildIR/Graph/GraphBuilder.swift +++ b/Sources/NativeBuilder/ContainerBuildIR/Graph/GraphBuilder.swift @@ -179,8 +179,20 @@ public final class GraphBuilder { network: NetworkMode = .default, ) throws -> Self { let cmd = shell ? Command.shell(command) : Command.exec(command.split(separator: " ").map(String.init)) - let envVars = env.map { (key: $0.key, value: EnvironmentValue.literal($0.value)) } + return try runWithCmd(cmd, shell: shell, env: env, workdir: workdir, user: user, mounts: mounts, network: network) + } + @discardableResult + public func runWithCmd( + _ cmd: Command, + shell: Bool = true, + env: [String: String] = [:], + workdir: String? = nil, + user: User? = nil, + mounts: [Mount] = [], + network: NetworkMode = .default, + ) throws -> Self { + let envVars = env.map { (key: $0.key, value: EnvironmentValue.literal($0.value)) } let operation = ExecOperation( command: cmd, environment: Environment(envVars), @@ -280,6 +292,14 @@ public final class GraphBuilder { return try add(operation) } + @discardableResult + public func labelBatch(labels: [String: String]) throws -> Self { + let operation = MetadataOperation( + action: .setLabelBatch(labels) + ) + return try add(operation) + } + /// Expose port @discardableResult public func expose(_ port: Int, protocolType: PortSpec.NetworkProtocol = .tcp) throws -> Self { diff --git a/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerInstructionVisitor.swift b/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerInstructionVisitor.swift index 6546cf62..114e58ac 100644 --- a/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerInstructionVisitor.swift +++ b/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerInstructionVisitor.swift @@ -20,6 +20,8 @@ protocol InstructionVisitor { func visit(_ from: FromInstruction) throws func visit(_ run: RunInstruction) throws func visit(_ copy: CopyInstruction) throws + func visit(_ cmd: CMDInstruction) throws + func visit(_ label: LabelInstruction) throws } /// DockerInstructionVisitor visits each provided DockerInstruction and builds a @@ -114,7 +116,7 @@ extension DockerInstructionVisitor { mounts.append(graphMount) } - try graphBuilder.run(run.command, shell: run.shell, mounts: mounts) + try graphBuilder.runWithCmd(run.command, mounts: mounts) } func visit(_ copy: CopyInstruction) throws { @@ -136,4 +138,12 @@ extension DockerInstructionVisitor { } try graphBuilder.copyFromContext(paths: copy.sources, to: copy.destination, chown: copy.chown, chmod: copy.chmod) } + + func visit(_ cmd: CMDInstruction) throws { + try graphBuilder.cmd(cmd.command) + } + + func visit(_ label: LabelInstruction) throws { + try graphBuilder.labelBatch(labels: label.labels) + } } diff --git a/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerfileParser.swift b/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerfileParser.swift index 09239fe7..96a9de7a 100644 --- a/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerfileParser.swift +++ b/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerfileParser.swift @@ -63,6 +63,12 @@ public struct DockerfileParser: BuildParser { return try tokensToFromInstruction(tokens: tokens) case .RUN: return try tokensToRunInstruction(tokens: tokens) + case .COPY: + return try tokensToCopyInstruction(tokens: tokens) + case .CMD: + return try tokensToCMDInstruction(tokens: tokens) + case .LABEL: + return try tokensToLabelInstruction(tokens: tokens) default: throw ParseError.invalidInstruction(value) } @@ -78,13 +84,13 @@ public struct DockerfileParser: BuildParser { // Step 1: parse options while index < tokens.endIndex { - guard case .option(let key, let value) = tokens[index] else { + guard case .option(let option) = tokens[index] else { break } - guard FromOptions(rawValue: key) == .platform else { + guard FromOptions(rawValue: option.key) == .platform else { throw ParseError.unexpectedValue } - platform = value + platform = option.value index += 1 } @@ -133,29 +139,42 @@ public struct DockerfileParser: BuildParser { // Step 1: parse options while index < tokens.endIndex { - guard case .option(let key, let value) = tokens[index] else { + guard case .option(let option) = tokens[index] else { break } - guard let option = RunOptions(rawValue: key) else { + guard let runOpt = RunOptions(rawValue: option.key) else { throw ParseError.unexpectedValue } - switch option { + switch runOpt { case .mount: - rawMounts.append(value) + rawMounts.append(option.value) case .network: - network = value + network = option.value default: throw ParseError.unexpectedValue } index += 1 } + // Step 2: parse run command + let (newIndex, cmd) = getCommand(start: index, tokens: tokens) + index = newIndex + + // check for extra tokens + if index < tokens.endIndex { + throw ParseError.unexpectedValue + } + + return try RunInstruction(command: cmd, rawMounts: rawMounts, network: network) + } + + private func getCommand(start: Int, tokens: [Token]) -> (index: Int, cmd: Command) { var command = [String]() var shell = true - // Step 2: parse run command and if we're using shell or exec form + var index = start while index < tokens.endIndex { if case .stringList(let value) = tokens[index], command.isEmpty { // when using the exec form, there should only be a single list for the command @@ -166,18 +185,16 @@ public struct DockerfileParser: BuildParser { break } else if case .stringLiteral(let value) = tokens[index] { command.append(value) + } else if case .option(let option) = tokens[index] { + command.append(option.raw) } else { break } index += 1 } - // check for extra tokens - if index < tokens.endIndex { - throw ParseError.unexpectedValue - } - - return try RunInstruction(command: command, shell: shell, rawMounts: rawMounts, network: network) + let cmd = shell ? Command.shell(command.joined(separator: " ")) : Command.exec(command) + return (index, cmd) } internal func tokensToCopyInstruction(tokens: [Token]) throws -> CopyInstruction { @@ -191,35 +208,35 @@ public struct DockerfileParser: BuildParser { // Step 1: parse options while index < tokens.endIndex { - guard case .option(let key, let value) = tokens[index] else { + guard case .option(let option) = tokens[index] else { break } - guard let option = CopyOptions(rawValue: key) else { + guard let copyOpt = CopyOptions(rawValue: option.key) else { throw ParseError.unexpectedValue } - switch option { + switch copyOpt { case .from: if from != nil { throw ParseError.duplicateOptionSet(CopyOptions.from.rawValue) } - from = value + from = option.value case .chown: if chown != nil { throw ParseError.duplicateOptionSet(CopyOptions.chown.rawValue) } - chown = value + chown = option.value case .chmod: if chmod != nil { throw ParseError.duplicateOptionSet(CopyOptions.chmod.rawValue) } - chmod = value + chmod = option.value case .link: if link != nil { throw ParseError.duplicateOptionSet(CopyOptions.link.rawValue) } - link = value + link = option.value default: throw ParseError.unexpectedValue } @@ -250,4 +267,48 @@ public struct DockerfileParser: BuildParser { return try CopyInstruction(sources: sources, destination: destination, from: from, ownership: chown, permissions: chmod) } + internal func tokensToCMDInstruction(tokens: [Token]) throws -> CMDInstruction { + var index = tokens.startIndex + index += 1 + + // get the command + let (newIndex, cmd) = getCommand(start: index, tokens: tokens) + index = newIndex + + // check for extra tokens + if index < tokens.endIndex { + throw ParseError.unexpectedValue + } + + return CMDInstruction(command: cmd) + } + + internal func tokensToLabelInstruction(tokens: [Token]) throws -> LabelInstruction { + var index = tokens.startIndex + index += 1 + + var labels: [String: String] = [:] + while index < tokens.endIndex { + guard case .stringLiteral(let option) = tokens[index] else { + break + } + let components = option.split(separator: "=", maxSplits: 1) + guard components.count == 2 else { + throw ParseError.unexpectedValue + } + let key = String(components[0]) + guard labels[key] == nil else { + throw ParseError.duplicateOptionSet(key) + } + labels[key] = String(components[1]) + index += 1 + } + + // check for extra tokens + if index < tokens.endIndex { + throw ParseError.unexpectedValue + } + + return LabelInstruction(labels: labels) + } } diff --git a/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerfileTokenizer.swift b/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerfileTokenizer.swift index 36e154cd..a3c13947 100644 --- a/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerfileTokenizer.swift +++ b/Sources/NativeBuilder/ContainerBuildParser/Docker/DockerfileTokenizer.swift @@ -135,10 +135,11 @@ struct DockerfileTokenizer { let valueStart = position parseWord() let rawValue = input[valueStart..