diff --git a/docs/sqlite-databases.md b/docs/sqlite-databases.md index 3b8f40b01f..274a93bbbb 100644 --- a/docs/sqlite-databases.md +++ b/docs/sqlite-databases.md @@ -105,6 +105,20 @@ overall native heap and total resident memory are not capped. Row limits do not override these byte and resource limits. Resource-limit failures do not return an incomplete schema as successful complete evidence. +If the combined database and WAL exceed the snapshot budget, the refusal is a +`resource_constraint` and reports both the configured byte limit and the exact +observed combined size as a decimal string. The remediation directs callers to +create a smaller consistent offline snapshot containing only relevant data +through the application's supported export workflow. A generic backup or copy +does not guarantee a smaller result; REA does not truncate or modify the +selected source. + +On hosts that cannot address the derived WAL or rollback-journal path for a +long selected database name, REA reports that the SQLite capability is +unavailable. It does not assume an inaccessible sidecar is absent. Use the +application's supported consistent offline export or copy workflow to place +the complete known file set at a shorter path. + MCP also applies its configured response budget to the complete Evidence envelope. If delivery exceeds that budget, it reports a transport constraint with guidance for exporting retained Evidence, requesting a matching larger diff --git a/scripts/lib/test-projects.mjs b/scripts/lib/test-projects.mjs index f284625cc4..2be464ec47 100644 --- a/scripts/lib/test-projects.mjs +++ b/scripts/lib/test-projects.mjs @@ -17,7 +17,7 @@ export const SOURCE_TEST_PROJECTS = [ group: "source", include: [ "src/*.test.ts", - "src/{apktool,jeb,adb,artifacts,browser,dotnet,electron,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts", + "src/{apktool,jeb,adb,artifacts,browser,dotnet,electron,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,sqlite,wasm,windows}/**/*.test.ts", ], }, { diff --git a/src/sqlite/SqliteDatabaseFailures.ts b/src/sqlite/SqliteDatabaseFailures.ts index 09a1c08677..2e67b94067 100644 --- a/src/sqlite/SqliteDatabaseFailures.ts +++ b/src/sqlite/SqliteDatabaseFailures.ts @@ -79,7 +79,7 @@ export const sqliteDatabaseFailure = ( return new AnalysisInputError(operation, { cause }, [ { path: ["path"], - reason: cause.reason === "limit" ? "out_of_range" : "invalid_format", + reason: "invalid_format", message: cause.message, }, ]); @@ -110,7 +110,12 @@ export const sqliteDatabaseFailure = ( "The selected SQLite database is held open by another process without read sharing. Close that application or inspect a copy, then retry.", }, ); - if (cause.code === "ENOENT" || cause.code === "ENOTDIR") + if ( + cause.code === "ENOENT" || + cause.code === "ENOTDIR" || + cause.code === "ELOOP" || + cause.code === "ENAMETOOLONG" + ) return new AnalysisInputError(operation, { cause }, [ { path: ["path"], diff --git a/src/sqlite/SqliteDatabaseSnapshot.ts b/src/sqlite/SqliteDatabaseSnapshot.ts index 1384fbdb6e..aa7112cfdf 100644 --- a/src/sqlite/SqliteDatabaseSnapshot.ts +++ b/src/sqlite/SqliteDatabaseSnapshot.ts @@ -3,7 +3,12 @@ import { constants, type BigIntStats } from "node:fs"; import { join } from "node:path"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; -import { AnalysisInputError } from "../domain/analysisErrorCore.js"; +import { + AnalysisCapabilityUnavailableError, + AnalysisInputError, + AnalysisResourceConstraintError, +} from "../domain/analysisErrorCore.js"; +import { SQLITE_PROVIDER_IDENTITY } from "../domain/sqlite/sqliteDatabase.js"; import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; import { readStableArtifact, @@ -15,6 +20,49 @@ const fileSystem: StableArtifactFileSystem = { lstat: (path) => lstat(path, { bigint: true }), open, }; + +/** Probe a generated sibling only after the caller-selected database is observed. */ +const optionalSidecarStat = async ( + path: string, + databasePath: string, + observedDatabase: BigIntStats, + member: "WAL" | "rollback journal", + selectedFileSystem: StableArtifactFileSystem, +): Promise => { + let observedSidecar: BigIntStats | null; + try { + observedSidecar = await optionalStat(path, selectedFileSystem); + } catch (cause: unknown) { + if ( + cause instanceof Error && + "code" in cause && + cause.code === "ENAMETOOLONG" + ) { + const currentDatabase = await duringAcquisition(databasePath, () => + optionalStat(databasePath, selectedFileSystem), + ); + if (!sameState(observedDatabase, currentDatabase)) + throw new ArtifactReaderFailure( + "integrity", + `SQLite snapshot file set changed before sidecar inspection: ${databasePath}`, + { cause }, + ); + const remedy = + "Use the application's supported consistent offline export or copy workflow to move the complete known database file set to a shorter path. Do not rename a live database or omit its WAL or journal."; + throw new AnalysisCapabilityUnavailableError( + SQLITE_PROVIDER_IDENTITY.id, + "inspect_sqlite_database", + `Cannot safely address the derived SQLite ${member} path (${String(cause.code)}): ${path}`, + { + cause, + userMessage: `The derived SQLite ${member} path is too long to inspect safely (ENAMETOOLONG). ${remedy}`, + }, + ); + } + throw cause; + } + return observedSidecar; +}; const optionalStat = async ( path: string, selectedFileSystem: StableArtifactFileSystem, @@ -50,13 +98,35 @@ const duringAcquisition = async ( if ( cause instanceof Error && "code" in cause && - ["ENOENT", "ENOTDIR", "ELOOP"].includes(String(cause.code)) + ["ENOENT", "ENOTDIR", "ELOOP", "ENAMETOOLONG"].includes( + String(cause.code), + ) ) throw new ArtifactReaderFailure( "integrity", `SQLite snapshot file set changed during acquisition: ${path}`, { cause }, ); + if (cause instanceof ArtifactReaderFailure && cause.reason === "limit") { + const changed = new ArtifactReaderFailure( + "integrity", + `SQLite snapshot file set exceeded its observed size during acquisition: ${path}`, + { + cause, + ...(cause.partialObservation === undefined + ? {} + : { partialObservation: cause.partialObservation }), + }, + cause.details, + ); + if (cause.cleanup !== undefined) + throw ArtifactReaderFailure.withCleanup( + changed, + cause.cleanup, + cause.partialObservation, + ); + throw changed; + } throw cause; } }; @@ -186,30 +256,50 @@ export const captureSqliteDatabaseSnapshot = async ( selectedFileSystem: StableArtifactFileSystem = fileSystem, ) => { signal?.throwIfAborted(); - const paths = [path, `${path}-wal`, `${path}-journal`]; - const before = await Promise.all( - paths.map((path) => optionalStat(path, selectedFileSystem)), - ); - const database = before[0]; - const wal = before[1]; - if (database === undefined || wal === undefined || before[2] === undefined) - throw new Error("Incomplete database file-set observation"); + const walPath = `${path}-wal`; + const journalPath = `${path}-journal`; + const paths = [path, walPath, journalPath] as const; + const database = await optionalStat(path, selectedFileSystem); if (database === null) throw missingDatabase(path); - if (before[2] !== null) + const wal = await optionalSidecarStat( + walPath, + path, + database, + "WAL", + selectedFileSystem, + ); + const journal = await optionalSidecarStat( + journalPath, + path, + database, + "rollback journal", + selectedFileSystem, + ); + const before = [database, wal, journal] as const; + const selectedSize = database.size + (wal?.size ?? 0n); + if (selectedSize > BigInt(SQLITE_DATABASE_LIMITS.inputBytes)) + throw new AnalysisResourceConstraintError( + "inspect_sqlite_database", + "file-size", + `Selected SQLite database and WAL total ${selectedSize.toString()} bytes, exceeding the combined ${String(SQLITE_DATABASE_LIMITS.inputBytes)}-byte offline snapshot budget: ${path}`, + { + sqlite_snapshot_input_bytes: SQLITE_DATABASE_LIMITS.inputBytes, + observed_sqlite_snapshot_input_bytes: selectedSize.toString(), + }, + { + remediationAction: + "Use the application's supported export workflow to create a consistent, smaller offline SQLite snapshot containing only the relevant data, then inspect that snapshot. REA will not truncate or modify the selected source.", + }, + ); + if (journal !== null) await verifyInactiveJournal( `${path}-journal`, - before[2], + journal, selectedFileSystem, resourceScope, "initial", signal, ); - const selectedSize = database.size + (wal?.size ?? 0n); - if (selectedSize > BigInt(SQLITE_DATABASE_LIMITS.inputBytes)) - throw new ArtifactReaderFailure( - "limit", - `Selected SQLite database and WAL exceed the combined ${String(SQLITE_DATABASE_LIMITS.inputBytes)}-byte snapshot budget: ${path}`, - ); const snapshotPath = join(root, "database.snapshot"); const captured = await readObservedArtifact( path, @@ -252,10 +342,10 @@ export const captureSqliteDatabaseSnapshot = async ( } // Identity alone does not establish an inactive journal when timestamps are // coarse. Re-read the header after copying, using the same validity owner. - if (before[2] !== null) + if (journal !== null) await verifyInactiveJournal( `${path}-journal`, - before[2], + journal, selectedFileSystem, resourceScope, "after-copy", diff --git a/src/sqlite/SqliteDatabaseSnapshotLimits.test.ts b/src/sqlite/SqliteDatabaseSnapshotLimits.test.ts new file mode 100644 index 0000000000..52c9d02ceb --- /dev/null +++ b/src/sqlite/SqliteDatabaseSnapshotLimits.test.ts @@ -0,0 +1,194 @@ +import { lstat, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import type { StableArtifactFileSystem } from "../artifacts/readStableArtifact.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import type { AnalysisPartialObservation } from "../domain/analysisErrorBase.js"; +import { captureSqliteDatabaseSnapshot } from "./SqliteDatabaseSnapshot.js"; +import { SQLITE_DATABASE_LIMITS } from "./SqliteDatabaseLimits.js"; +import { sqliteDatabaseFailure } from "./SqliteDatabaseFailures.js"; + +it("reports filesystem sizes above the safe-integer range exactly before opening files", async () => { + const databaseBytes = BigInt(Number.MAX_SAFE_INTEGER) + 1n; + const walBytes = 17n; + const workspace = await mkdtemp(join(tmpdir(), "rea-sqlite-budget-")); + const databasePath = join(workspace, "selected.db"); + const walPath = `${databasePath}-wal`; + const journalPath = `${databasePath}-journal`; + const privateRoot = join(workspace, "private"); + await writeFile(databasePath, "tiny fixture"); + const stat = await lstat(databasePath, { bigint: true }); + const open = async (): Promise => { + throw new Error("An over-budget source must be refused before open"); + }; + const fileSystem: StableArtifactFileSystem = { + lstat: async (path) => { + if (path === journalPath) + throw Object.assign(new Error("journal absent"), { code: "ENOENT" }); + const size = + path === databasePath + ? databaseBytes + : path === walPath + ? walBytes + : stat.size; + return Object.assign( + Object.create(Object.getPrototypeOf(stat)) as typeof stat, + stat, + { size }, + ); + }, + open, + }; + const resourceScope = new ArtifactResourceScope(); + + try { + const failure = await captureSqliteDatabaseSnapshot( + databasePath, + privateRoot, + resourceScope, + undefined, + fileSystem, + ).then( + () => { + throw new Error("Expected the oversized snapshot to be refused"); + }, + (cause: unknown) => cause, + ); + const projected = projectAnalysisError( + sqliteDatabaseFailure(failure, databasePath, "artifact-read"), + ); + expect(failure).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + operation: "inspect_sqlite_database", + resource: "file-size", + reportedLimits: { + sqlite_snapshot_input_bytes: SQLITE_DATABASE_LIMITS.inputBytes, + observed_sqlite_snapshot_input_bytes: ( + databaseBytes + walBytes + ).toString(), + }, + remediationAction: expect.stringContaining( + "smaller offline SQLite snapshot", + ), + }); + expect(projected).toMatchObject({ + code: "resource_constraint", + category: "resource_constraint", + retryable: false, + details: { + reported_limits: { + sqlite_snapshot_input_bytes: SQLITE_DATABASE_LIMITS.inputBytes, + observed_sqlite_snapshot_input_bytes: ( + databaseBytes + walBytes + ).toString(), + }, + }, + remediation: { + action: expect.stringContaining("supported export workflow"), + }, + }); + await expect(readdir(privateRoot)).rejects.toMatchObject({ + code: "ENOENT", + }); + } finally { + await resourceScope.close(); + await rm(workspace, { force: true, recursive: true }); + } +}); + +it.each([ + { label: "incomplete cleanup", retainCleanup: true }, + { label: "partial observation without cleanup", retainCleanup: false }, +])( + "preserves $label when a late read-limit failure becomes changed evidence", + async ({ retainCleanup }) => { + const workspace = await mkdtemp(join(tmpdir(), "rea-sqlite-read-cleanup-")); + const databasePath = join(workspace, "selected.db"); + await writeFile(databasePath, "tiny fixture"); + const stat = await lstat(databasePath, { bigint: true }); + const cleanup = { + reason: "fixture file-handle close failed", + resources: [databasePath], + }; + const partialObservation = { + operation: "address_to_file_offset", + address: "0x1000", + provider_file_offset: 16, + provider_source_path: null, + provider_image_header_hex: null, + original_file_offset: { + available: false, + reason: "fixture observation", + }, + } satisfies AnalysisPartialObservation; + const details = { + logicalPath: "selected.db", + declaredSha256: null, + calculatedSha256: null, + unpacked: false, + }; + const primary = new ArtifactReaderFailure( + "limit", + "Selected artifact grew beyond its byte limit", + { partialObservation }, + details, + ); + const limitFailure = retainCleanup + ? ArtifactReaderFailure.withCleanup(primary, cleanup) + : primary; + const fileSystem: StableArtifactFileSystem = { + lstat: async (path) => { + if (path === `${databasePath}-journal`) + throw Object.assign(new Error("journal absent"), { code: "ENOENT" }); + return stat; + }, + open: async () => { + throw limitFailure; + }, + }; + const resourceScope = new ArtifactResourceScope(); + + try { + const failure = await captureSqliteDatabaseSnapshot( + databasePath, + join(workspace, "private"), + resourceScope, + undefined, + fileSystem, + ).then( + () => { + throw new Error("Expected the changed snapshot to be rejected"); + }, + (cause: unknown) => cause, + ); + expect(failure).toMatchObject({ reason: "integrity" }); + if (retainCleanup) + expect(failure).toMatchObject({ + cleanup, + cause: expect.any(ArtifactReaderFailure), + }); + expect(failure).toMatchObject({ partialObservation, details }); + const projectedFailure = sqliteDatabaseFailure( + failure, + databasePath, + "artifact-read", + ); + if (retainCleanup) { + expect(projectedFailure).toBeInstanceOf(ProviderCleanupError); + expect(projectedFailure).toMatchObject({ + cleanupIncomplete: true, + cleanupResources: cleanup.resources, + }); + } + } finally { + await resourceScope.close(); + await rm(workspace, { force: true, recursive: true }); + } + }, +); diff --git a/tests/acceptance/sqliteDatabaseCli.test.ts b/tests/acceptance/sqliteDatabaseCli.test.ts index d23c032a6e..d0ffb361c0 100644 --- a/tests/acceptance/sqliteDatabaseCli.test.ts +++ b/tests/acceptance/sqliteDatabaseCli.test.ts @@ -1,5 +1,13 @@ import { createHash } from "node:crypto"; -import { readFile, stat, writeFile } from "node:fs/promises"; +import { + lstat, + readFile, + rename, + stat, + symlink, + truncate, + writeFile, +} from "node:fs/promises"; import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { expect, onTestFinished } from "vitest"; @@ -20,6 +28,165 @@ import { cliTest } from "../support/cli/cliFixture.js"; const digest = (bytes: Buffer): string => createHash("sha256").update(bytes).digest("hex"); +for (const expectedErrno of ["ELOOP", "ENAMETOOLONG"] as const) { + cliTest.skipIf(expectedErrno === "ELOOP" && process.platform === "win32")( + `reports an initial ${expectedErrno} path failure through CLI and MCP`, + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-path-failure-"); + const path = + expectedErrno === "ELOOP" + ? join(root, "cycle-a", "selected.db") + : join(root, "a".repeat(300), "selected.db"); + if (expectedErrno === "ELOOP") { + await symlink("cycle-b", join(root, "cycle-a")); + await symlink("cycle-a", join(root, "cycle-b")); + } + const result = await cli.run({ + arguments: ["inspect-sqlite-database", path, "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode).toBe(1); + const expectedError = { + code: "invalid_request", + details: { + issues: [ + { + path: ["path"], + reason: "invalid_value", + message: expect.stringContaining(expectedErrno), + }, + ], + }, + }; + expect(result.json).toMatchObject(expectedError); + const { call } = await connectLocalToolsMcp(); + const response = await call("inspect_sqlite_database", { path }); + expect(parseMcpToolError(response).error).toMatchObject(expectedError); + }, + ); +} + +cliTest( + "reports the combined database and WAL byte budget through CLI and MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-size-budget-"); + const path = join(root, "selected.db"); + const walPath = `${path}-wal`; + const maximum = 256 * 1024 * 1024; + const databaseBytes = maximum / 2; + const walBytes = databaseBytes + 1; + await writeFile(path, ""); + await writeFile(walPath, ""); + await truncate(path, databaseBytes); + await truncate(walPath, walBytes); + const before = await Promise.all( + [path, walPath].map(async (filePath) => ({ + path: filePath, + state: await lstat(filePath, { bigint: true }), + })), + ); + const cliResult = await cli.run({ + arguments: ["inspect-sqlite-database", path, "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + const { call } = await connectLocalToolsMcp(); + const mcpResponse = await call("inspect_sqlite_database", { path }); + const mcpError = parseMcpToolError(mcpResponse).error; + const expectedError = { + code: "resource_constraint", + category: "resource_constraint", + retryable: false, + details: { + reported_limits: { + sqlite_snapshot_input_bytes: maximum, + observed_sqlite_snapshot_input_bytes: String(maximum + 1), + }, + }, + remediation: { + action: expect.stringContaining("supported export workflow"), + }, + }; + expect(cliResult.exitCode).toBe(1); + expect(cliResult.json).toMatchObject(expectedError); + expect(mcpError).toMatchObject(expectedError); + const after = await Promise.all( + before.map(async ({ path: filePath }) => ({ + path: filePath, + state: await lstat(filePath, { bigint: true }), + })), + ); + for (const [index, original] of before.entries()) { + const current = after[index]; + expect(current?.state).toMatchObject({ + dev: original.state.dev, + ino: original.state.ino, + mode: original.state.mode, + size: original.state.size, + mtimeNs: original.state.mtimeNs, + ctimeNs: original.state.ctimeNs, + }); + } + }, +); + +cliTest.skipIf(process.platform === "win32" || !SQLITE_NATIVE_LIMITS_AVAILABLE)( + "reports an unavailable capability when a closed database sidecar path is too long", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-long-basename-"); + const path = join(root, "x".repeat(255)); + const originalPath = join(root, "selected.db"); + const database = new DatabaseSync(originalPath); + try { + database.exec( + "CREATE TABLE selected(value TEXT); INSERT INTO selected VALUES ('retained');", + ); + } finally { + database.close(); + } + await rename(originalPath, path); + const selectedBefore = await lstat(path, { bigint: true }); + await expect(lstat(`${path}-wal`)).rejects.toMatchObject({ + code: "ENAMETOOLONG", + }); + await expect(lstat(`${path}-journal`)).rejects.toMatchObject({ + code: "ENAMETOOLONG", + }); + + const cliResult = await cli.run({ + arguments: ["inspect-sqlite-database", path, "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + const { call } = await connectLocalToolsMcp(); + const mcpResult = await call("inspect_sqlite_database", { path }); + const expectedError = { + code: "capability_unavailable", + category: "unsupported_provider", + retryable: false, + message: expect.stringContaining("supported consistent offline export"), + details: { + provider_id: "sqlite", + operation: "inspect_sqlite_database", + reason: expect.stringContaining("ENAMETOOLONG"), + }, + remediation: { + action: expect.stringContaining("supported consistent offline export"), + }, + }; + expect(cliResult.exitCode).toBe(1); + expect(cliResult.json).toMatchObject(expectedError); + expect(parseMcpToolError(mcpResult).error).toMatchObject(expectedError); + const selectedAfter = await lstat(path, { bigint: true }); + expect(selectedAfter).toMatchObject({ + dev: selectedBefore.dev, + ino: selectedBefore.ino, + mode: selectedBefore.mode, + size: selectedBefore.size, + mtimeNs: selectedBefore.mtimeNs, + ctimeNs: selectedBefore.ctimeNs, + }); + }, +); + cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( "bounds default context and projects exact columns without evaluating unrelated cells", async ({ cli }) => { diff --git a/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts b/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts index 4fce636ceb..3687ffc72a 100644 --- a/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseSnapshot.test.ts @@ -121,6 +121,50 @@ it.runIf(process.platform !== "win32").each([ }, ); +it.each(["ELOOP", "ENAMETOOLONG"] as const)( + "treats %s during file acquisition as changed evidence, not an invalid input path", + async (code) => { + const { database, capture } = await fixture(); + const fileSystem: StableArtifactFileSystem = { + lstat: (path) => lstat(path, { bigint: true }), + open: async (path, flags) => { + if (path === database.path) + throw Object.assign( + new Error(`${code}: path changed after observation`), + { code }, + ); + return open(path, flags); + }, + }; + + await expect(capture(fileSystem)).rejects.toMatchObject({ + reason: "integrity", + message: expect.stringContaining(database.path), + }); + }, +); + +it("retains changed-evidence semantics when a sidecar path failure follows database removal", async () => { + const { database, capture } = await fixture(); + const fileSystem: StableArtifactFileSystem = { + lstat: async (path) => { + if (path === database.walPath) { + await unlink(database.path); + throw Object.assign(new Error("derived sidecar path is too long"), { + code: "ENAMETOOLONG", + }); + } + return lstat(path, { bigint: true }); + }, + open, + }; + + await expect(capture(fileSystem)).rejects.toMatchObject({ + reason: "integrity", + message: expect.stringContaining(database.path), + }); +}); + it("refuses an active journal even before SQLite writes its magic header", async () => { const { database, capture } = await fixture(); const writer = new DatabaseSync(database.path);