From 2ae4e1608f290c95ff7cff28c4d16a79b301d03d Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 18 Oct 2026 09:40:02 +0800 Subject: [PATCH] fix(cli): preserve omitted positions in filtered JSON arrays Normalize only filter-owned selections before streamed JSON and JSONL output, and count the same normalized payload. Keep source values and strict domain JSON validation intact. Refs #4938. --- src/cli/cliResultOutput.ts | 36 +++- .../boundary/cli/filteredArrayOutput.test.ts | 169 ++++++++++++++++++ 2 files changed, 200 insertions(+), 5 deletions(-) create mode 100644 tests/boundary/cli/filteredArrayOutput.test.ts diff --git a/src/cli/cliResultOutput.ts b/src/cli/cliResultOutput.ts index 3cc98d8664..93cb3e5d34 100644 --- a/src/cli/cliResultOutput.ts +++ b/src/cli/cliResultOutput.ts @@ -226,9 +226,18 @@ export const createCliResultOutput = ( metadata, originalFailure: cliOperationFailureIdentity(value), }; - const filtered: unknown = options.filterOutput - ? Filter.apply(value, Filter.parse(options.filterOutput)) - : value; + const filterPaths = options.filterOutput + ? Filter.parse(options.filterOutput) + : undefined; + const filtered: unknown = + filterPaths === undefined ? value : Filter.apply(value, filterPaths); + if ( + filterPaths !== undefined && + filterPaths.length > 0 && + (format === "json" || format === "jsonl") && + Array.isArray(filtered) + ) + normalizeFilteredJsonArrays(filtered); // Leave successful YAML and bare YAML rendering to Incur. The custom // boundary is needed only for a full failure envelope, whose error // sentinel otherwise drops category, remediation, and details. @@ -257,8 +266,7 @@ export const createCliResultOutput = ( } } if (characters <= constants.MAX_STRING_LENGTH) { - if (failure === undefined) return false; - const count = estimateTokenCount(Formatter.format(failure, format)); + const count = estimateTokenCount(Formatter.format(tokenValue, format)); handled = true; try { await writeOutputText(destination, `${count}\n`); @@ -364,3 +372,21 @@ const fullFailureEnvelope = ( meta: { ...meta, ...(nextOffset === undefined ? {} : { nextOffset }) }, }; }; + +/** Filter.apply owns these mapped arrays; an empty path list returns the source. */ +const normalizeFilteredJsonArrays = (selection: unknown[]): void => { + const pending = [selection]; + while (pending.length > 0) { + const array = pending.pop()!; + for (let index = 0; index < array.length; index += 1) { + const value: unknown = array[index]; + if ( + value === undefined || + typeof value === "function" || + typeof value === "symbol" + ) + array[index] = null; + else if (Array.isArray(value)) pending.push(value); + } + } +}; diff --git a/tests/boundary/cli/filteredArrayOutput.test.ts b/tests/boundary/cli/filteredArrayOutput.test.ts new file mode 100644 index 0000000000..52deb77eb1 --- /dev/null +++ b/tests/boundary/cli/filteredArrayOutput.test.ts @@ -0,0 +1,169 @@ +import { Writable } from "node:stream"; + +import { Cli, Formatter } from "incur"; +import { estimateTokenCount } from "tokenx"; +import { expect, it } from "vitest"; + +import { + createCliResultOutput, + streamCliCommandResults, +} from "../../../src/cli/cliResultOutput.js"; + +const source = [{ selected: 1 }, { other: 2 }, { selected: null }]; + +it.each( + (["json", "jsonl"] as const).flatMap((format) => + [false, true].map((full) => ({ format, full })), + ), +)( + "preserves filtered array positions in $format (full: $full)", + async ({ format, full }) => { + const value = Object.freeze( + source.map((item) => Object.freeze({ ...item })), + ); + const cli = Cli.create("filtered-array-test", { sync: false }); + cli.command("rows", { run: () => value }); + const args = [ + "rows", + "--format", + format, + "--filter-output", + "selected", + ...(full ? ["--full-output"] : []), + ]; + const chunks: Buffer[] = []; + const output = createCliResultOutput( + args, + new Writable({ + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }), + ); + if (output === undefined) throw new Error("Missing CLI output owner"); + streamCliCommandResults(cli, output); + let fallback = ""; + let status = 0; + await cli.serve(args, { + stdout: (text) => { + if (!output.handled) fallback += text; + }, + exit: (code) => { + status = code; + }, + }); + const text = Buffer.concat(chunks).toString("utf8"); + if (full) + expect(JSON.parse(text)).toMatchObject({ + ok: true, + data: [1, null, null], + meta: { command: "rows" }, + }); + else if (format === "json") + expect(JSON.parse(text)).toEqual([1, null, null]); + else expect(text).toBe("1\nnull\nnull\n"); + expect(fallback).toBe(""); + expect(status).toBe(0); + expect(output.handled).toBe(true); + expect(output.failed).toBe(false); + expect(value).toEqual(source); + }, +); + +it.each(["json", "jsonl"] as const)( + "counts the normalized filtered %s records", + async (format) => { + const cli = Cli.create("filtered-count-test", { sync: false }); + cli.command("rows", { run: () => source }); + const args = [ + "rows", + "--format", + format, + "--filter-output", + "selected", + "--token-count", + ]; + const chunks: Buffer[] = []; + const output = createCliResultOutput( + args, + new Writable({ + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }), + ); + if (output === undefined) throw new Error("Missing CLI output owner"); + streamCliCommandResults(cli, output); + let fallback = ""; + await cli.serve(args, { + exit: (code) => { + expect(code).toBe(0); + }, + stdout: (text) => { + if (!output.handled) fallback += text; + }, + }); + expect(Buffer.concat(chunks).toString()).toBe( + `${estimateTokenCount(Formatter.format([1, null, null], format))}\n`, + ); + expect(fallback).toBe(""); + }, +); + +it.each(["selected", "toString", "constructor"])( + "normalizes only owned nested selection arrays for %s", + async (filter) => { + const value = [[{ selected: false }, {}], [{ selected: 0 }], []]; + const before = structuredClone(value); + const chunks: Buffer[] = []; + const output = createCliResultOutput( + ["--json", "--filter-output", filter], + new Writable({ + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }), + ); + await output?.write(value, { + command: "rows", + duration: "0ms", + format: "json", + }); + expect(JSON.parse(Buffer.concat(chunks).toString())).toEqual( + filter === "selected" + ? [[false, null], [0], []] + : [[null, null], [null], []], + ); + expect(value).toEqual(before); + }, +); + +it("finishes JSONL after a bounded large record before a missing selection", async () => { + const value = [{ selected: "x".repeat(80 * 1024) }, {}]; + const chunks: Buffer[] = []; + const output = createCliResultOutput( + ["--format", "jsonl", "--filter-output", "selected"], + new Writable({ + highWaterMark: 1, + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk); + setImmediate(callback); + }, + }), + ); + if (output === undefined) throw new Error("Missing CLI output owner"); + await output.write(value, { + command: "rows", + duration: "0ms", + format: "jsonl", + }); + const lines = Buffer.concat(chunks).toString().trimEnd().split("\n"); + expect(lines.map((line) => JSON.parse(line))).toEqual([ + value[0]?.selected, + null, + ]); + expect(output.failed).toBe(false); +});