feat(sqlite): make worker deadlines adjustable with recovery guidance (#5818)
Expose a per-call SQLite worker deadline with exact CLI/MCP recovery controls, parent-owned execution metadata and historical unknown preservation. Keep fixed resource limits and cleanup admission intact. Share typed cleanup composition with one captured-output owner and preserve the timeout after cleanup resolves. Fixes #5812. Follow-up tracking: #5808. Existing CLI test-oracle failure: #5845. Verified 155 SQLite tests (3 skipped), static/docs/discovery checks, six real CLI/MCP pairs and 13 actual Codex sessions including four corrected deadline recoveries. Three pre-dispatch discovery failures remain explicit, not passes. Review findings include real retained-cleanup and fixture ownership regressions. Assisted-by: Kogasa (gensokyo-skills:kogasa-surprise-testing)
This commit is contained in:
1 parent
5241597594
commit
f801b2fbfd
15 files changed
+496
-49
No files matched your search
@@ -102,8 +102,29 @@ bounds simultaneous input buffering and private snapshot/worker counts.
|
||||
|
||||
The database and WAL together are limited to 256 MiB. SQLite values/records,
|
||||
SQL statements and the complete worker JSON reply each have a 16 MiB limit.
|
||||
The owned worker has a 256 MiB JavaScript heap limit and a 30-second inspection
|
||||
deadline. SQLite native allocations are separate from the JavaScript heap;
|
||||
The owned worker has a 256 MiB JavaScript heap limit and defaults to a 30,000 ms
|
||||
deadline. Select optional MCP `worker_timeout_ms`, or CLI
|
||||
`--worker-timeout-ms <milliseconds>`, to change the budget for that call. Values
|
||||
must be integer milliseconds from 1 to 2,147,483,647 (the runtime timer range).
|
||||
It covers worker startup, queries and reply writing, **not** DB/WAL snapshot
|
||||
acquisition, final reply reads, cleanup or delivery. Allow corresponding headroom
|
||||
in the MCP client's whole-call deadline; increasing only that deadline does not
|
||||
change the worker budget. No server restart is needed.
|
||||
|
||||
The result's `worker_timeout_ms` reports the actual parent-supervised budget,
|
||||
including the default. Historical Evidence without this field has unknown
|
||||
execution settings, not an inferred default. Explicit input selection is also
|
||||
retained in Evidence parameters; omission stays omitted. Completed observations
|
||||
retained under cancellation or cleanup failure keep the same budget facts.
|
||||
Timeout errors name the selected milliseconds, exact CLI/MCP controls, and
|
||||
cleanup-before-retry requirements. Incomplete cleanup remains a blocker even
|
||||
when a larger budget is available.
|
||||
|
||||
For example, `rea inspect-sqlite-database ./application.sqlite
|
||||
--worker-timeout-ms 60000 --json` selects 60 seconds; equivalent MCP input is
|
||||
`{"path":"/artifacts/application.sqlite","worker_timeout_ms":60000}`.
|
||||
|
||||
SQLite native allocations are separate from the JavaScript heap;
|
||||
overall native heap and total resident memory are not capped. Row limits do not
|
||||
override these byte and resource limits. Resource-limit failures do not return
|
||||
an incomplete schema as successful complete evidence.
|
||||
|
||||
@@ -4,11 +4,13 @@ import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCo
|
||||
import { err } from "../../domain/result.js";
|
||||
import { ok } from "../../domain/result.js";
|
||||
import { createAnalysisExecution } from "../AnalysisProvider.js";
|
||||
import { sqliteDatabaseSchema } from "../../domain/sqlite/sqliteDatabase.js";
|
||||
|
||||
const fixture = () => ({
|
||||
artifact: { path: "/selected.db", sha256: "a".repeat(64), bytes: 4096 },
|
||||
wal: null,
|
||||
engine: { name: "SQLite", version: "3.50.2" },
|
||||
worker_timeout_ms: 30_000,
|
||||
database_encoding: "UTF-8",
|
||||
schema: {
|
||||
scope: "inventory",
|
||||
@@ -43,6 +45,10 @@ it.each([
|
||||
table: "data",
|
||||
row_limit: Number.MAX_SAFE_INTEGER + 1,
|
||||
},
|
||||
...[0, -1, 1.5, 2_147_483_648, "1000", null].map((worker_timeout_ms) => ({
|
||||
path: "/selected.db",
|
||||
worker_timeout_ms,
|
||||
})),
|
||||
])(
|
||||
"rejects invalid SQLite selection before provider effects: %j",
|
||||
async (input) => {
|
||||
@@ -58,6 +64,70 @@ it.each([
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves the selected worker budget in provider input and Evidence", async () => {
|
||||
const report = { ...fixture(), worker_timeout_ms: 1234 };
|
||||
const service = new SqliteDatabaseService({
|
||||
inspect: async (input) => {
|
||||
expect(input.worker_timeout_ms).toBe(1234);
|
||||
return ok(execution(report));
|
||||
},
|
||||
});
|
||||
const result = await service.inspect({
|
||||
path: "/selected.db",
|
||||
worker_timeout_ms: 1234,
|
||||
});
|
||||
if (!result.ok) throw result.error;
|
||||
expect(result.value.parameters).toMatchObject({ worker_timeout_ms: 1234 });
|
||||
expect(result.value.normalized_result).toMatchObject({
|
||||
worker_timeout_ms: 1234,
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ selected: undefined, reported: undefined },
|
||||
{ selected: undefined, reported: 1234 },
|
||||
{ selected: 1234, reported: undefined },
|
||||
{ selected: 1234, reported: 30_000 },
|
||||
])(
|
||||
"rejects a missing or changed fresh producer budget: %j",
|
||||
async ({ selected, reported }) => {
|
||||
const { worker_timeout_ms: _default, ...report } = fixture();
|
||||
const service = new SqliteDatabaseService({
|
||||
inspect: async () =>
|
||||
ok(
|
||||
execution({
|
||||
...report,
|
||||
...(reported === undefined ? {} : { worker_timeout_ms: reported }),
|
||||
}),
|
||||
),
|
||||
});
|
||||
expect(
|
||||
await service.inspect({
|
||||
path: "/selected.db",
|
||||
...(selected === undefined ? {} : { worker_timeout_ms: selected }),
|
||||
}),
|
||||
).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } });
|
||||
},
|
||||
);
|
||||
|
||||
it("retains the actual default without inventing an explicit caller selection", async () => {
|
||||
const service = new SqliteDatabaseService({
|
||||
inspect: async () => ok(execution(fixture())),
|
||||
});
|
||||
const result = await service.inspect({ path: "/selected.db" });
|
||||
if (!result.ok) throw result.error;
|
||||
expect(result.value.normalized_result).toMatchObject({
|
||||
worker_timeout_ms: 30_000,
|
||||
});
|
||||
expect(result.value.parameters).not.toHaveProperty("worker_timeout_ms");
|
||||
});
|
||||
|
||||
it("parses historical report omissions as unknown without supplying a default", () => {
|
||||
const { worker_timeout_ms: _default, ...historical } = fixture();
|
||||
const parsed = sqliteDatabaseSchema.parse(historical);
|
||||
expect(parsed).not.toHaveProperty("worker_timeout_ms");
|
||||
});
|
||||
|
||||
it("preserves producer failures and rejects pre-cancelled requests before effects", async () => {
|
||||
const failure = new AnalysisCapabilityUnavailableError(
|
||||
"rea-sqlite",
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
type InspectSqliteDatabaseInput,
|
||||
type SqliteDatabase,
|
||||
SQLITE_ROW_LIMIT_DEFAULT,
|
||||
DEFAULT_SQLITE_WORKER_TIMEOUT_MS,
|
||||
sqliteSchemaScope,
|
||||
sqliteInspectionParameters,
|
||||
} from "../../domain/sqlite/sqliteDatabase.js";
|
||||
@@ -36,6 +37,9 @@ const matchesSelectedDatabase = (
|
||||
subject.path === input.path &&
|
||||
subject.sha256 === report.artifact.sha256 &&
|
||||
subject.format === "file" &&
|
||||
// Fresh executions must report known settings; only historical parsing permits omission.
|
||||
report.worker_timeout_ms ===
|
||||
(input.worker_timeout_ms ?? DEFAULT_SQLITE_WORKER_TIMEOUT_MS) &&
|
||||
(report.wal === null || report.wal.path === `${input.path}-wal`) &&
|
||||
report.schema.scope === sqliteSchemaScope(input) &&
|
||||
(report.schema.scope !== "selected_table" ||
|
||||
@@ -105,7 +109,7 @@ export class SqliteDatabaseService {
|
||||
return err(
|
||||
new AnalysisOutputError(
|
||||
OPERATION,
|
||||
"SQLite adapter returned malformed data or changed the selected database identity/table/row limit.",
|
||||
"SQLite adapter returned malformed data or changed the selected database identity/table/row limit/worker budget.",
|
||||
),
|
||||
);
|
||||
return ok(
|
||||
|
||||
@@ -7,6 +7,7 @@ import type { CliInstance } from "./types.js";
|
||||
import { withCommandCancellation } from "./commandCancellation.js";
|
||||
import { absoluteCallerPath } from "../filesystem/callerPath.js";
|
||||
import { finishAnalysisCommand } from "./finishAnalysisCommand.js";
|
||||
import { inspectSqliteDatabaseInputSchema } from "../domain/sqlite/sqliteDatabase.js";
|
||||
|
||||
/** Resolve an operator-selected SQLite snapshot through the shared inspection workflow. */
|
||||
export const registerSqliteCommands = (
|
||||
@@ -22,6 +23,7 @@ export const registerSqliteCommands = (
|
||||
path: z.string().describe("Local SQLite database snapshot"),
|
||||
}),
|
||||
options: z.object({
|
||||
workerTimeoutMs: inspectSqliteDatabaseInputSchema.shape.worker_timeout_ms,
|
||||
table: z
|
||||
.string()
|
||||
.optional()
|
||||
@@ -55,6 +57,9 @@ export const registerSqliteCommands = (
|
||||
service.inspect(
|
||||
{
|
||||
path: absoluteCallerPath(args.path),
|
||||
...(options.workerTimeoutMs === undefined
|
||||
? {}
|
||||
: { worker_timeout_ms: options.workerTimeoutMs }),
|
||||
...(options.table === undefined
|
||||
? {}
|
||||
: { table: options.table }),
|
||||
|
||||
@@ -13,7 +13,7 @@ export const SQLITE_TOOL_CONTRACTS = [
|
||||
...toolContractMetadata("inspect_sqlite_database"),
|
||||
kind: "artifact-provider",
|
||||
description:
|
||||
"Inspect an explicit local SQLite database snapshot and its existing sibling WAL without modifying source files or opening a native analysis provider. Requires Node.js 24.x >=24.15 or 26+ with native DatabaseSync.limits support, checked by the worker. Returns observed Evidence with database/WAL SHA-256 and a compact schema inventory. Selecting an exact ordinary table expands only its schema and samples rows (default 20, subject to byte and worker resource limits). Select exact columns in the requested order to avoid unrelated or large values; schema_scope=database explicitly expands the whole schema; returned rows retain 64-bit integers, text bytes and blobs with explicit truncation. No arbitrary SQL, view or virtual-table execution, extension loading, database repair, encryption support or live-snapshot atomicity claim. At most two inspections per provider are admitted; occupied capacity is retryable. An isolated temporary copy and owned worker enforce input, value/record, SQL-statement, JSON-reply, JavaScript-heap and deadline limits; overall native heap is not capped. Supply a quiescent snapshot for coherent database/WAL evidence.",
|
||||
"Inspect an explicit local SQLite database snapshot and its existing sibling WAL without modifying source files or opening a native analysis provider. Requires Node.js 24.x >=24.15 or 26+ with native DatabaseSync.limits support, checked by the worker. Returns observed Evidence with database/WAL SHA-256 and a compact schema inventory. Selecting an exact ordinary table expands only its schema and samples rows (default 20, subject to byte and worker resource limits). Select exact columns in the requested order to avoid unrelated or large values; schema_scope=database explicitly expands the whole schema; returned rows retain 64-bit integers, text bytes and blobs with explicit truncation. Adjust worker_timeout_ms (default 30000 ms) for worker startup, queries and reply writing; allow separate client headroom for snapshot acquisition, reply reads, cleanup and delivery. The result retains the actual budget; byte/heap safety limits stay fixed. No arbitrary SQL, view or virtual-table execution, extension loading, database repair, encryption support or live-snapshot atomicity claim. At most two inspections per provider are admitted; occupied capacity is retryable. An isolated temporary copy and owned worker enforce input, value/record, SQL-statement, JSON-reply, JavaScript-heap and deadline limits; overall native heap is not capped. Supply a quiescent snapshot for coherent database/WAL evidence.",
|
||||
inputSchema: inspectSqliteDatabaseInputSchema,
|
||||
outputSchema: evidenceResultOf(sqliteDatabaseSchema),
|
||||
examples: [
|
||||
|
||||
@@ -11,6 +11,10 @@ export const SQLITE_PROVIDER_IDENTITY = {
|
||||
|
||||
/** Default ordinary-table sample size when the caller selects a table. */
|
||||
export const SQLITE_ROW_LIMIT_DEFAULT = 20;
|
||||
/** Execution budget, bounded by the runtime's signed 32-bit timer range. */
|
||||
export const DEFAULT_SQLITE_WORKER_TIMEOUT_MS = 30_000;
|
||||
export const MAX_SQLITE_WORKER_TIMEOUT_MS = 2_147_483_647;
|
||||
const workerTimeout = z.number().int().min(1).max(MAX_SQLITE_WORKER_TIMEOUT_MS);
|
||||
|
||||
const nonnegative = z.number().int().nonnegative();
|
||||
const hex = z.string().regex(/^(?:[0-9a-f]{2})*$/);
|
||||
@@ -27,6 +31,11 @@ export const inspectSqliteDatabaseInputSchema = z
|
||||
path: localPathStringSchema.describe(
|
||||
"Absolute path to an offline SQLite database snapshot",
|
||||
),
|
||||
worker_timeout_ms: workerTimeout
|
||||
.optional()
|
||||
.describe(
|
||||
`Deadline in milliseconds for worker startup, queries and reply writing (default ${String(DEFAULT_SQLITE_WORKER_TIMEOUT_MS)}), not snapshot acquisition or the whole call. Allow client headroom for acquisition, reply reads, cleanup and delivery. Fixed byte/heap limits do not change.`,
|
||||
),
|
||||
table: z
|
||||
.string()
|
||||
.optional()
|
||||
@@ -114,6 +123,11 @@ export const sqliteDatabaseSchema = z
|
||||
name: z.literal("SQLite"),
|
||||
version: z.string().min(1),
|
||||
}),
|
||||
worker_timeout_ms: workerTimeout
|
||||
.exactOptional()
|
||||
.describe(
|
||||
"Actual parent-supervised worker deadline in milliseconds; absent in historical Evidence means unknown, not the default.",
|
||||
),
|
||||
database_encoding: encoding,
|
||||
schema: z.strictObject({
|
||||
scope: z.enum(["inventory", "selected_table", "database"]),
|
||||
@@ -220,6 +234,9 @@ export const sqliteInspectionParameters = (
|
||||
input: InspectSqliteDatabaseInput,
|
||||
): Readonly<Record<string, JsonValue>> => ({
|
||||
path: input.path,
|
||||
...(input.worker_timeout_ms === undefined
|
||||
? {}
|
||||
: { worker_timeout_ms: input.worker_timeout_ms }),
|
||||
...(input.table === undefined ? {} : { table: input.table }),
|
||||
...(input.columns === undefined ? {} : { columns: input.columns }),
|
||||
...(input.schema_scope === undefined
|
||||
|
||||
@@ -8,7 +8,9 @@ import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
|
||||
import {
|
||||
AnalysisError,
|
||||
type AnalysisCapturedOutput,
|
||||
type AnalysisErrorOptions,
|
||||
} from "../domain/analysisErrorBase.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import {
|
||||
AnalysisAccessDeniedError,
|
||||
AnalysisArtifactChangedError,
|
||||
@@ -23,8 +25,11 @@ import { ProviderAdapterError } from "../domain/providerAdapterError.js";
|
||||
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
|
||||
import { OwnedCommandFailure } from "../process/OwnedCommand.js";
|
||||
import type { ProviderProcessSnapshot } from "../process/ProviderProcess.js";
|
||||
import { SQLITE_PROVIDER_IDENTITY } from "../domain/sqlite/sqliteDatabase.js";
|
||||
import { SQLITE_DATABASE_LIMITS } from "./SqliteDatabaseLimits.js";
|
||||
import {
|
||||
SQLITE_PROVIDER_IDENTITY,
|
||||
DEFAULT_SQLITE_WORKER_TIMEOUT_MS,
|
||||
MAX_SQLITE_WORKER_TIMEOUT_MS,
|
||||
} from "../domain/sqlite/sqliteDatabase.js";
|
||||
|
||||
/** Preserve diagnostics independently of worker exit and private-root cleanup. */
|
||||
export const capturedSqliteOutput = (
|
||||
@@ -35,6 +40,38 @@ export const capturedSqliteOutput = (
|
||||
truncated: snapshot.diagnosticTruncated === true,
|
||||
});
|
||||
|
||||
/** Keep typed cleanup history and one canonical output owner, including after resolution. */
|
||||
export const createSqliteCleanupError = (
|
||||
resources: readonly string[],
|
||||
diagnostics: Readonly<Record<string, JsonValue>>,
|
||||
options: Omit<AnalysisErrorOptions, "diagnostics">,
|
||||
): ProviderCleanupError => {
|
||||
const capturedOutput =
|
||||
options.capturedOutput ?? options.cleanupPrimary?.capturedOutput;
|
||||
let previous =
|
||||
options.cleanupPrimary === undefined
|
||||
? null
|
||||
: projectAnalysisError(options.cleanupPrimary);
|
||||
// History carries facts/recovery; the wrapper delivers output only once.
|
||||
if (capturedOutput !== undefined && previous?.details !== undefined) {
|
||||
const { captured_output: _capturedOutput, ...metadata } = previous.details;
|
||||
previous = { ...previous, details: metadata };
|
||||
}
|
||||
return new ProviderCleanupError(
|
||||
SQLITE_PROVIDER_IDENTITY.id,
|
||||
resources,
|
||||
{
|
||||
...diagnostics,
|
||||
previous_error: previous ?? diagnostics.previous_error ?? null,
|
||||
},
|
||||
{
|
||||
...options,
|
||||
operation: "inspect_sqlite_database",
|
||||
...(capturedOutput === undefined ? {} : { capturedOutput }),
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
/** The file-set member a filesystem error names, if it is the database, its WAL or its journal. */
|
||||
const busyMember = (cause: Error, path: string): string => {
|
||||
const failed = "path" in cause ? cause.path : undefined;
|
||||
@@ -50,6 +87,7 @@ export const sqliteDatabaseFailure = (
|
||||
path: string,
|
||||
phase: string,
|
||||
signal?: AbortSignal,
|
||||
workerTimeoutMs = DEFAULT_SQLITE_WORKER_TIMEOUT_MS,
|
||||
): AnalysisError => {
|
||||
const operation = "inspect_sqlite_database";
|
||||
if (cause instanceof AnalysisError) return cause;
|
||||
@@ -61,28 +99,36 @@ export const sqliteDatabaseFailure = (
|
||||
: undefined;
|
||||
let previous: AnalysisError | undefined;
|
||||
if (physicalRead?.kind === "failed")
|
||||
previous = sqliteDatabaseFailure(physicalRead.cause, path, phase);
|
||||
previous = sqliteDatabaseFailure(
|
||||
physicalRead.cause,
|
||||
path,
|
||||
phase,
|
||||
undefined,
|
||||
workerTimeoutMs,
|
||||
);
|
||||
else if (cause.reason === "cancelled")
|
||||
previous = new AnalysisCancelledError(operation, {
|
||||
cause: artifactFileCancellationCause(cause),
|
||||
});
|
||||
else if (physicalRead?.kind === "completed") previous = undefined;
|
||||
else if (cause.cause !== undefined)
|
||||
previous = sqliteDatabaseFailure(cause.cause, path, phase);
|
||||
previous = sqliteDatabaseFailure(
|
||||
cause.cause,
|
||||
path,
|
||||
phase,
|
||||
undefined,
|
||||
workerTimeoutMs,
|
||||
);
|
||||
else if (signal?.aborted)
|
||||
previous = new AnalysisCancelledError(operation, {
|
||||
cause: signal.reason,
|
||||
});
|
||||
const error = new ProviderCleanupError(
|
||||
SQLITE_PROVIDER_IDENTITY.id,
|
||||
const error = createSqliteCleanupError(
|
||||
cause.cleanup.resources,
|
||||
{
|
||||
reason: cause.cleanup.reason,
|
||||
previous_error:
|
||||
previous === undefined ? null : projectAnalysisError(previous),
|
||||
},
|
||||
{
|
||||
operation,
|
||||
cause,
|
||||
...(previous === undefined ? {} : { cleanupPrimary: previous }),
|
||||
},
|
||||
@@ -91,7 +137,13 @@ export const sqliteDatabaseFailure = (
|
||||
return error;
|
||||
}
|
||||
if (cause.reason === "io" && cause.cause !== undefined)
|
||||
return sqliteDatabaseFailure(cause.cause, path, phase, signal);
|
||||
return sqliteDatabaseFailure(
|
||||
cause.cause,
|
||||
path,
|
||||
phase,
|
||||
signal,
|
||||
workerTimeoutMs,
|
||||
);
|
||||
if (cause.reason === "integrity")
|
||||
return new AnalysisArtifactChangedError(operation, path, cause.message, {
|
||||
cause,
|
||||
@@ -116,7 +168,7 @@ export const sqliteDatabaseFailure = (
|
||||
: cause,
|
||||
});
|
||||
if (cause instanceof OwnedCommandFailure) {
|
||||
const projected = ownedCommandFailure(cause);
|
||||
const projected = ownedCommandFailure(cause, workerTimeoutMs);
|
||||
if (projected !== undefined) return projected;
|
||||
}
|
||||
if (phase === "artifact-read" && cause instanceof Error && "code" in cause) {
|
||||
@@ -173,6 +225,7 @@ export const sqliteDatabaseFailure = (
|
||||
|
||||
const ownedCommandFailure = (
|
||||
cause: OwnedCommandFailure,
|
||||
workerTimeoutMs: number,
|
||||
): AnalysisError | undefined => {
|
||||
const operation = "inspect_sqlite_database";
|
||||
const capturedOutput =
|
||||
@@ -181,21 +234,37 @@ const ownedCommandFailure = (
|
||||
cause,
|
||||
...(capturedOutput === undefined ? {} : { capturedOutput }),
|
||||
};
|
||||
// Preserve the same typed primary whether or not owned cleanup also fails.
|
||||
const timeout =
|
||||
cause.reason === "timeout"
|
||||
? new AnalysisTimeoutError(operation, workerTimeoutMs, {
|
||||
...options,
|
||||
userMessage: `The SQLite worker reached its ${String(workerTimeoutMs)} ms server/provider deadline (worker_timeout_ms). This is not a whole-inspection or MCP client deadline.`,
|
||||
remediationAction: `${
|
||||
workerTimeoutMs < MAX_SQLITE_WORKER_TIMEOUT_MS
|
||||
? `Increase worker_timeout_ms above ${String(workerTimeoutMs)} (integer milliseconds, maximum ${String(MAX_SQLITE_WORKER_TIMEOUT_MS)}), or use rea inspect-sqlite-database <path> --worker-timeout-ms <milliseconds> on the CLI.`
|
||||
: `worker_timeout_ms is already at the maximum supported ${String(MAX_SQLITE_WORKER_TIMEOUT_MS)} ms; no higher worker deadline is supported. Report the blocker or narrow the question with caller approval.`
|
||||
} This per-call setting needs no server restart. It covers worker startup, queries and reply writing, not snapshot acquisition or final reply reads/cleanup. Allow client headroom for the whole call; increasing only the client deadline does not change the worker budget. Keep the selected database/WAL and requested facts; narrow table/columns/schema_scope/row_limit only if the question permits it. Byte and heap safety limits remain fixed. Resolve incomplete cleanup before retrying; do not repeat the unchanged failed call.`,
|
||||
})
|
||||
: undefined;
|
||||
if (cause.cleanupFailure !== null)
|
||||
return new ProviderCleanupError(
|
||||
SQLITE_PROVIDER_IDENTITY.id,
|
||||
return createSqliteCleanupError(
|
||||
cause.resources,
|
||||
{
|
||||
reason: cause.cleanupFailure,
|
||||
previous_error: {
|
||||
failure_kind: cause.reason,
|
||||
message: cause.message,
|
||||
},
|
||||
...(capturedOutput === undefined
|
||||
? {}
|
||||
: { captured_output: { ...capturedOutput } }),
|
||||
...(timeout === undefined
|
||||
? {
|
||||
previous_error: {
|
||||
failure_kind: cause.reason,
|
||||
message: cause.message,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
{
|
||||
...options,
|
||||
...(timeout === undefined ? {} : { cleanupPrimary: timeout }),
|
||||
},
|
||||
{ operation, cause },
|
||||
);
|
||||
if (cause.reason === "cancelled")
|
||||
return new AnalysisCancelledError(operation, {
|
||||
@@ -203,12 +272,7 @@ const ownedCommandFailure = (
|
||||
cause:
|
||||
cause.cancellation === undefined ? cause : cause.cancellation.reason,
|
||||
});
|
||||
if (cause.reason === "timeout")
|
||||
return new AnalysisTimeoutError(
|
||||
operation,
|
||||
SQLITE_DATABASE_LIMITS.timeoutMs,
|
||||
options,
|
||||
);
|
||||
if (timeout !== undefined) return timeout;
|
||||
if (cause.reason === "output-limit")
|
||||
return new AnalysisOutputError(operation, cause.message, options);
|
||||
if (
|
||||
|
||||
@@ -549,7 +549,7 @@ export const inspectSqliteDatabaseSnapshot = (
|
||||
"SQLite reports current schema and selected committed rows from its valid database/WAL state. Deleted records, uncommitted or invalid WAL tails, encrypted databases, forensic recovery, and full integrity verification are outside this profile.",
|
||||
"Stored CREATE statements preserve index/view/trigger definitions. Views, triggers, and virtual/shadow table rows are not executed; virtual-table column expansion is unsupported.",
|
||||
"Selected rows have unspecified order and are a bounded sample. Text display may replace invalid sequences; bytes_hex and encoding preserve SQLite's text byte representation, not original on-disk page offsets.",
|
||||
"Resource boundaries: at most two concurrent inspections per provider; combined input 256 MiB per inspection, complete worker JSON 16 MiB, native SQLite value/record and SQL statement lengths 16 MiB, owned-worker JavaScript heap 256 MiB and supervised deadline 30 seconds. These are distinct from total resident memory; an overall native SQLite heap cap is not claimed. Failures return no partial success.",
|
||||
"Resource boundaries: at most two concurrent inspections per provider; combined input 256 MiB per inspection, complete worker JSON 16 MiB, native SQLite value/record and SQL statement lengths 16 MiB, owned-worker JavaScript heap 256 MiB. The parent applies a separate supervised worker deadline, reported in worker_timeout_ms. These are distinct from total resident memory; an overall native SQLite heap cap is not claimed. Failures return no partial success.",
|
||||
];
|
||||
const budget = new OutputBudget();
|
||||
const { schema, selectedTable } = readSchema(db, input, {
|
||||
|
||||
@@ -3,7 +3,6 @@ export const SQLITE_DATABASE_LIMITS = {
|
||||
concurrentInspections: 2,
|
||||
inputBytes: 256 * 1024 * 1024,
|
||||
outputBytes: 16 * 1024 * 1024,
|
||||
timeoutMs: 30_000,
|
||||
diagnosticBytes: 1024 * 1024,
|
||||
} as const;
|
||||
/** Internal rejection retains the failed boundary without returning partial success. */
|
||||
|
||||
@@ -27,6 +27,7 @@ import {
|
||||
sqliteDatabaseSchema,
|
||||
type InspectSqliteDatabaseInput,
|
||||
SQLITE_PROVIDER_IDENTITY,
|
||||
DEFAULT_SQLITE_WORKER_TIMEOUT_MS,
|
||||
sqliteInspectionParameters,
|
||||
} from "../domain/sqlite/sqliteDatabase.js";
|
||||
import { isAbsolute } from "node:path";
|
||||
@@ -39,7 +40,10 @@ import type {
|
||||
ProviderProcessStopResult,
|
||||
ProviderProcessSupervisor,
|
||||
} from "../process/ProviderProcess.js";
|
||||
import { sqliteDatabaseFailure } from "./SqliteDatabaseFailures.js";
|
||||
import {
|
||||
createSqliteCleanupError,
|
||||
sqliteDatabaseFailure,
|
||||
} from "./SqliteDatabaseFailures.js";
|
||||
import { captureSqliteDatabaseSnapshot } from "./SqliteDatabaseSnapshot.js";
|
||||
import { SQLITE_DATABASE_LIMITS } from "./SqliteDatabaseLimits.js";
|
||||
|
||||
@@ -257,6 +261,8 @@ export class SqliteDatabaseProvider implements SqliteDatabasePort {
|
||||
let retainedOutput: AnalysisCapturedOutput | undefined;
|
||||
let phase = "configuration";
|
||||
let result: Result<AnalysisExecution, AnalysisError>;
|
||||
const workerTimeoutMs =
|
||||
input.worker_timeout_ms ?? DEFAULT_SQLITE_WORKER_TIMEOUT_MS;
|
||||
try {
|
||||
if (options?.signal?.aborted)
|
||||
throw new AnalysisCancelledError(operation, {
|
||||
@@ -283,6 +289,7 @@ export class SqliteDatabaseProvider implements SqliteDatabasePort {
|
||||
input,
|
||||
root: root.path,
|
||||
snapshotPath: snapshot.snapshotPath,
|
||||
timeoutMs: workerTimeoutMs,
|
||||
environment: this.environment,
|
||||
resourceScope: this.#artifactResources,
|
||||
...(this.launcher === undefined ? {} : { launcher: this.launcher }),
|
||||
@@ -299,6 +306,7 @@ export class SqliteDatabaseProvider implements SqliteDatabasePort {
|
||||
artifact: snapshot.artifact,
|
||||
wal: snapshot.wal,
|
||||
...inspection,
|
||||
worker_timeout_ms: workerTimeoutMs,
|
||||
});
|
||||
if (!validated.success)
|
||||
throw new AnalysisOutputError(
|
||||
@@ -322,6 +330,7 @@ export class SqliteDatabaseProvider implements SqliteDatabasePort {
|
||||
input.path,
|
||||
phase,
|
||||
options?.signal,
|
||||
workerTimeoutMs,
|
||||
);
|
||||
retainedOutput ??= failure.capturedOutput;
|
||||
result = err(failure);
|
||||
@@ -495,17 +504,14 @@ const sqliteCleanupFailure = (
|
||||
const partialObservation = result.ok
|
||||
? completedSqliteEvidence(input, result.value)
|
||||
: result.error.partialObservation;
|
||||
return new ProviderCleanupError(
|
||||
SQLITE_PROVIDER_IDENTITY.id,
|
||||
return createSqliteCleanupError(
|
||||
resources,
|
||||
{
|
||||
...details,
|
||||
reason,
|
||||
previous_error: result.ok ? null : projectAnalysisError(result.error),
|
||||
...(output === undefined ? {} : { captured_output: { ...output } }),
|
||||
},
|
||||
{
|
||||
operation,
|
||||
...(output === undefined ? {} : { capturedOutput: output }),
|
||||
...(result.ok ? {} : { cleanupPrimary: result.error }),
|
||||
...(cause === undefined ? {} : { cause }),
|
||||
...(partialObservation === undefined ? {} : { partialObservation }),
|
||||
|
||||
@@ -8,7 +8,6 @@ import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { z } from "zod";
|
||||
import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js";
|
||||
import type { AnalysisCapturedOutput } from "../domain/analysisErrorBase.js";
|
||||
import {
|
||||
AnalysisCancelledError,
|
||||
@@ -36,6 +35,7 @@ const operation = "inspect_sqlite_database";
|
||||
const {
|
||||
artifact: _artifactSchema,
|
||||
wal: _walSchema,
|
||||
worker_timeout_ms: _workerTimeoutSchema,
|
||||
...inspectionShape
|
||||
} = sqliteDatabaseSchema.shape;
|
||||
const inspectionSchema = z.strictObject(inspectionShape);
|
||||
@@ -101,7 +101,9 @@ const rejectedReply = (
|
||||
},
|
||||
{
|
||||
capturedOutput,
|
||||
...fixedAllowanceOptions("The SQLite worker reply allowance", "report"),
|
||||
retryable: false,
|
||||
remediationAction:
|
||||
"The SQLite value/record and worker JSON reply allowances are fixed at 16 MiB; worker_timeout_ms and client deadlines do not raise them. If it preserves the requested facts, select exact columns, reduce row_limit, or use schema_scope: inventory instead of database. This worker refusal leaves no completed Evidence to retrieve. Do not repeat the unchanged request or truncate required database bytes.",
|
||||
},
|
||||
);
|
||||
const argument = rejectedArgument[reply.reason];
|
||||
@@ -123,6 +125,7 @@ export const executeSqliteDatabaseWorker = async (request: {
|
||||
readonly input: InspectSqliteDatabaseInput;
|
||||
readonly root: string;
|
||||
readonly snapshotPath: string;
|
||||
readonly timeoutMs: number;
|
||||
readonly environment: Readonly<NodeJS.ProcessEnv>;
|
||||
readonly resourceScope: ArtifactResourceScope;
|
||||
readonly launcher?: SqliteWorkerLauncher;
|
||||
@@ -160,7 +163,7 @@ export const executeSqliteDatabaseWorker = async (request: {
|
||||
hostEnvironment: environment,
|
||||
},
|
||||
{
|
||||
timeoutMs: SQLITE_DATABASE_LIMITS.timeoutMs,
|
||||
timeoutMs: request.timeoutMs,
|
||||
diagnosticBytes: SQLITE_DATABASE_LIMITS.diagnosticBytes,
|
||||
},
|
||||
{
|
||||
|
||||
@@ -86,11 +86,20 @@ cliTest(
|
||||
})),
|
||||
);
|
||||
const cliResult = await cli.run({
|
||||
arguments: ["inspect-sqlite-database", path, "--json"],
|
||||
arguments: [
|
||||
"inspect-sqlite-database",
|
||||
path,
|
||||
"--worker-timeout-ms",
|
||||
"60000",
|
||||
"--json",
|
||||
],
|
||||
environment: { REA_LOG_LEVEL: "silent" },
|
||||
});
|
||||
const { call } = await connectLocalToolsMcp();
|
||||
const mcpResponse = await call("inspect_sqlite_database", { path });
|
||||
const mcpResponse = await call("inspect_sqlite_database", {
|
||||
path,
|
||||
worker_timeout_ms: 60000,
|
||||
});
|
||||
const mcpError = parseMcpToolError(mcpResponse).error;
|
||||
const expectedError = {
|
||||
code: "resource_constraint",
|
||||
@@ -240,10 +249,13 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
|
||||
path,
|
||||
table: "selected",
|
||||
columns: ["", "id", "id"],
|
||||
worker_timeout_ms: 42000,
|
||||
});
|
||||
expect(selected.isError).not.toBe(true);
|
||||
expect(selected.structuredContent).toMatchObject({
|
||||
parameters: { worker_timeout_ms: 42000 },
|
||||
normalized_result: {
|
||||
worker_timeout_ms: 42000,
|
||||
schema: {
|
||||
scope: "selected_table",
|
||||
tables: [{ name: "selected" }],
|
||||
@@ -283,13 +295,19 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
|
||||
"",
|
||||
"--row-limit",
|
||||
"1",
|
||||
"--worker-timeout-ms",
|
||||
"42000",
|
||||
"--json",
|
||||
],
|
||||
environment: { REA_LOG_LEVEL: "silent" },
|
||||
});
|
||||
expect(cliResult.exitCode, cliResult.stdout + cliResult.stderr).toBe(0);
|
||||
expect(cliResult.json).toMatchObject({
|
||||
normalized_result: { rows: { columns: ["id", ""], returned_rows: 1 } },
|
||||
parameters: { worker_timeout_ms: 42000 },
|
||||
normalized_result: {
|
||||
worker_timeout_ms: 42000,
|
||||
rows: { columns: ["id", ""], returned_rows: 1 },
|
||||
},
|
||||
});
|
||||
for (const input of [
|
||||
{ path, columns: ["id"] },
|
||||
|
||||
@@ -296,7 +296,16 @@ it("retains reply-read cancellation and retries the failed reply descriptor clos
|
||||
code: "cleanup_incomplete",
|
||||
details: {
|
||||
resources: [dirname(descriptorCleanup.path)],
|
||||
diagnostics: { previous_error: cancelled },
|
||||
captured_output: cancelled.details.captured_output,
|
||||
diagnostics: {
|
||||
previous_error: {
|
||||
code: cancelled.code,
|
||||
details: {
|
||||
execution_failure: "cancelled",
|
||||
resources: cancelled.details.resources,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
: cancelled,
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { OwnedCommandFailure } from "../../../src/process/OwnedCommand.js";
|
||||
import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js";
|
||||
import { sqliteDatabaseFailure } from "../../../src/sqlite/SqliteDatabaseFailures.js";
|
||||
|
||||
it.each([null, "owned stop unconfirmed"])(
|
||||
"preserves a selected worker deadline with cleanup=%s",
|
||||
(cleanup) => {
|
||||
const cause = new OwnedCommandFailure(
|
||||
"timeout",
|
||||
"Command deadline elapsed.",
|
||||
{
|
||||
stdout: { text: "partial-owned-output", bytes: 20, observedBytes: 20 },
|
||||
stderr: { text: "", bytes: 0, observedBytes: 0 },
|
||||
exitCode: null,
|
||||
signal: null,
|
||||
},
|
||||
cleanup,
|
||||
undefined,
|
||||
["pid:owned-fixture"],
|
||||
);
|
||||
const failure = sqliteDatabaseFailure(
|
||||
cause,
|
||||
"/selected.db",
|
||||
"worker",
|
||||
undefined,
|
||||
1234,
|
||||
);
|
||||
expect(failure.cause).toBe(cause);
|
||||
const projected = projectAnalysisError(failure);
|
||||
if (cleanup === null)
|
||||
expect(projected).toMatchObject({
|
||||
code: "provider_timeout",
|
||||
details: { timeout_ms: 1234 },
|
||||
remediation: { action: expect.stringContaining("--worker-timeout-ms") },
|
||||
});
|
||||
else
|
||||
expect(projected).toMatchObject({
|
||||
code: "cleanup_incomplete",
|
||||
retryable: false,
|
||||
details: {
|
||||
resources: ["pid:owned-fixture"],
|
||||
diagnostics: {
|
||||
previous_error: {
|
||||
code: "provider_timeout",
|
||||
details: { timeout_ms: 1234 },
|
||||
remediation: {
|
||||
action: expect.stringContaining("worker_timeout_ms"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(
|
||||
JSON.stringify(projected).split("partial-owned-output"),
|
||||
).toHaveLength(2);
|
||||
if (cleanup !== null) {
|
||||
expect(failure.cleanupPrimary).toMatchObject({
|
||||
_tag: "AnalysisTimeoutError",
|
||||
timeoutMs: 1234,
|
||||
});
|
||||
failure.resolveCleanup();
|
||||
const resolved = projectAnalysisError(failure);
|
||||
expect(resolved).toMatchObject({
|
||||
code: "provider_timeout",
|
||||
details: { timeout_ms: 1234 },
|
||||
});
|
||||
expect(
|
||||
JSON.stringify(resolved).split("partial-owned-output"),
|
||||
).toHaveLength(2);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("does not recommend a timer above the supported maximum", () => {
|
||||
const failure = projectAnalysisError(
|
||||
sqliteDatabaseFailure(
|
||||
new OwnedCommandFailure("timeout", "deadline"),
|
||||
"/selected.db",
|
||||
"worker",
|
||||
undefined,
|
||||
2_147_483_647,
|
||||
),
|
||||
);
|
||||
expect(failure.details).toMatchObject({ timeout_ms: 2_147_483_647 });
|
||||
expect(failure.remediation.action).toContain("no higher worker deadline");
|
||||
expect(failure.remediation.action).not.toContain("Increase");
|
||||
});
|
||||
@@ -51,6 +51,135 @@ const createFixture = async (wal = false) => {
|
||||
const expectPrivateRootRemoved = (path: string) =>
|
||||
expect(access(path)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
|
||||
it("enforces a selected worker budget and completes the corrected same-database retry", async () => {
|
||||
const { database } = await createFixture();
|
||||
const original = await readFile(database.path);
|
||||
let first = true;
|
||||
let root = "";
|
||||
const provider = new SqliteDatabaseProvider(process.env, (spawn) => {
|
||||
root = spawn.cwd ?? "";
|
||||
if (first) {
|
||||
first = false;
|
||||
return spawnOwnedProviderProcess({
|
||||
...spawn,
|
||||
arguments: [
|
||||
spawn.arguments[0] ?? "--max-old-space-size=256",
|
||||
"-e",
|
||||
`process.argv[2]=process.argv[1];setTimeout(()=>import(${JSON.stringify(workerUrl.href)}),250)`,
|
||||
spawn.arguments[2] ?? "",
|
||||
],
|
||||
});
|
||||
}
|
||||
return spawnOwnedProviderProcess({
|
||||
...spawn,
|
||||
arguments: spawn.arguments.map((arg, index) =>
|
||||
index === 1 ? fileURLToPath(workerUrl) : arg,
|
||||
),
|
||||
});
|
||||
});
|
||||
onTestFinished(() => provider.close());
|
||||
const failed = await provider.inspect({
|
||||
path: database.path,
|
||||
worker_timeout_ms: 50,
|
||||
});
|
||||
if (failed.ok) throw new Error("Worker deadline was ignored");
|
||||
expect(projectAnalysisError(failed.error)).toMatchObject({
|
||||
code: "provider_timeout",
|
||||
details: { timeout_ms: 50 },
|
||||
});
|
||||
expect(failed.error.cleanupIncomplete).toBe(false);
|
||||
await expectPrivateRootRemoved(root);
|
||||
const corrected = await provider.inspect({
|
||||
path: database.path,
|
||||
worker_timeout_ms: 2000,
|
||||
});
|
||||
if (!corrected.ok) throw corrected.error;
|
||||
expect(corrected.value.result).toMatchObject({
|
||||
worker_timeout_ms: 2000,
|
||||
schema: { completeness: "complete" },
|
||||
});
|
||||
await expectPrivateRootRemoved(root);
|
||||
expect(await readFile(database.path)).toEqual(original);
|
||||
});
|
||||
|
||||
nativeTest(
|
||||
"retains a timed-out worker's budget and one output copy through owned cleanup retry",
|
||||
async () => {
|
||||
const { database } = await createFixture();
|
||||
const original = await readFile(database.path);
|
||||
let cleanupAllowed = false;
|
||||
let launches = 0;
|
||||
let root = "";
|
||||
const provider = new SqliteDatabaseProvider(process.env, async (spawn) => {
|
||||
launches += 1;
|
||||
root = spawn.cwd ?? "";
|
||||
const launched = await spawnOwnedProviderProcess({
|
||||
...spawn,
|
||||
arguments: [
|
||||
"-e",
|
||||
"process.stdout.write('ready\\n');setTimeout(()=>process.stdout.write('sqlite-timeout-marker'),100);setInterval(()=>{},1000)",
|
||||
],
|
||||
});
|
||||
return {
|
||||
...launched,
|
||||
cleanup: async () =>
|
||||
cleanupAllowed
|
||||
? await (launched.cleanup?.() ??
|
||||
cleanupOwnedProcessGroup(launched.ownership))
|
||||
: { cleaned: false, reason: "owned timeout cleanup denied" },
|
||||
};
|
||||
});
|
||||
onTestFinished(async () => {
|
||||
cleanupAllowed = true;
|
||||
await provider.close();
|
||||
});
|
||||
const result = await provider.inspect({
|
||||
path: database.path,
|
||||
worker_timeout_ms: 3000,
|
||||
});
|
||||
if (result.ok) throw new Error("Expected timeout with unresolved cleanup");
|
||||
const projected = projectAnalysisError(result.error);
|
||||
expect(projected).toMatchObject({
|
||||
code: "cleanup_incomplete",
|
||||
retryable: false,
|
||||
details: {
|
||||
resources: expect.arrayContaining([root]),
|
||||
diagnostics: {
|
||||
previous_error: {
|
||||
code: "provider_timeout",
|
||||
details: { timeout_ms: 3000 },
|
||||
remediation: {
|
||||
action: expect.stringContaining("worker_timeout_ms"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(
|
||||
JSON.stringify(projected).split("sqlite-timeout-marker"),
|
||||
).toHaveLength(2);
|
||||
await expect(access(root)).resolves.toBeUndefined();
|
||||
expect(await provider.inspect({ path: database.path })).toMatchObject({
|
||||
ok: false,
|
||||
error: { cleanupIncomplete: true },
|
||||
});
|
||||
expect(launches).toBe(1);
|
||||
cleanupAllowed = true;
|
||||
await provider.close();
|
||||
const resolved = projectAnalysisError(result.error);
|
||||
expect(resolved).toMatchObject({
|
||||
code: "provider_timeout",
|
||||
details: { timeout_ms: 3000 },
|
||||
});
|
||||
expect(
|
||||
JSON.stringify(resolved).split("sqlite-timeout-marker"),
|
||||
).toHaveLength(2);
|
||||
expect(result.error.cleanupResources).toEqual([]);
|
||||
await expectPrivateRootRemoved(root);
|
||||
expect(await readFile(database.path)).toEqual(original);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["completed reply", "malformed reply"] as const)(
|
||||
"preserves the primary result for a %s while retaining unconfirmed cleanup ownership",
|
||||
async (replyKind) => {
|
||||
@@ -291,7 +420,13 @@ it.each(["late-cancellation", "cleanup-failure"])(
|
||||
await provider.close();
|
||||
});
|
||||
const result = await provider.inspect(
|
||||
{ path: database.path, table: "records", columns: ["id"], row_limit: 1 },
|
||||
{
|
||||
path: database.path,
|
||||
table: "records",
|
||||
columns: ["id"],
|
||||
row_limit: 1,
|
||||
worker_timeout_ms: 42000,
|
||||
},
|
||||
{ signal: controller.signal },
|
||||
);
|
||||
if (scenario === "late-cancellation") {
|
||||
@@ -313,8 +448,12 @@ it.each(["late-cancellation", "cleanup-failure"])(
|
||||
table: "records",
|
||||
columns: ["id"],
|
||||
row_limit: 1,
|
||||
worker_timeout_ms: 42000,
|
||||
},
|
||||
normalized_result: {
|
||||
worker_timeout_ms: 42000,
|
||||
rows: { columns: ["id"], returned_rows: 1 },
|
||||
},
|
||||
normalized_result: { rows: { columns: ["id"], returned_rows: 1 } },
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -338,8 +477,12 @@ it.each(["late-cancellation", "cleanup-failure"])(
|
||||
table: "records",
|
||||
columns: ["id"],
|
||||
row_limit: 1,
|
||||
worker_timeout_ms: 42000,
|
||||
},
|
||||
normalized_result: {
|
||||
worker_timeout_ms: 42000,
|
||||
rows: { columns: ["id"], returned_rows: 1 },
|
||||
},
|
||||
normalized_result: { rows: { columns: ["id"], returned_rows: 1 } },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user