Files
puter/src/backend/drivers/util/fileInput.test.ts
T
Daniel Salazar 4e4948ee22 fix(fs): keep S3 read stream errors from crashing the process
getObjectStream now attaches an 'error' listener to every body it returns.
A body that fails before its consumer attaches (socket timeout, reset, or a
checksum mismatch on a truncated read) used to emit 'error' with no listener
and exit the process, failing every in-flight request on the task. It now
logs a warning; consumers that attach later still receive the error.
2026-10-13 19:55:26 -04:00

603 lines
22 KiB
TypeScript

/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { GetObjectCommand } from '@aws-sdk/client-s3';
import { Readable } from 'node:stream';
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
import { v4 as uuidv4 } from 'uuid';
import type { Actor } from '../../core/actor.js';
import { runWithContext } from '../../core/context.js';
import { PuterServer } from '../../server.js';
import { setupTestServer } from '../../testUtil.js';
import { generateDefaultFsentries } from '../../util/userProvisioning.js';
import {
inferFilenameFromUrlOrPath,
loadFileInput,
resolveFileInput,
} from './fileInput.js';
// Web inputs leave the process through secureFetch; stub that boundary so
// the tests control the remote response without real network access.
const { secureFetchMock } = vi.hoisted(() => ({ secureFetchMock: vi.fn() }));
vi.mock('../../util/secureHttp.js', async (importOriginal) => ({
...(await importOriginal<typeof import('../../util/secureHttp.js')>()),
secureFetch: secureFetchMock,
}));
// ── Test harness ────────────────────────────────────────────────────
//
// Boots one real PuterServer (in-memory sqlite + dynamo + s3 + mock
// redis) and exercises `loadFileInput` against the live wired stores
// and FSService. Each test makes its own user via `makeUser` and,
// where the FS-resolution path is being exercised, writes a real
// file through FSService.write so there's an actual fsentry +
// in-memory S3 object behind the path / uuid we hand to loadFileInput.
let server: PuterServer;
beforeAll(async () => {
server = await setupTestServer();
});
afterAll(async () => {
await server?.shutdown();
});
const makeUser = async (): Promise<{ actor: Actor; userId: number }> => {
const username = `fic-${Math.random().toString(36).slice(2, 10)}`;
const created = await server.stores.user.create({
username,
uuid: uuidv4(),
password: null,
email: `${username}@test.local`,
free_storage: 100 * 1024 * 1024,
requires_email_confirmation: false,
});
await generateDefaultFsentries(
server.clients.db,
server.stores.user,
created,
);
const refreshed = (await server.stores.user.getById(created.id))!;
return {
userId: refreshed.id,
actor: {
user: {
id: refreshed.id,
uuid: refreshed.uuid,
username: refreshed.username,
email: refreshed.email ?? null,
email_confirmed: true,
} as Actor['user'],
},
};
};
const withActor = async <T>(actor: Actor, fn: () => Promise<T>): Promise<T> =>
runWithContext({ actor }, fn);
const writeFile = async (
userId: number,
path: string,
body: Buffer,
contentType = 'application/octet-stream',
) => {
const result = await server.services.fs.write(userId, {
fileMetadata: {
path,
size: body.byteLength,
contentType,
},
fileContent: body,
});
return result.fsEntry;
};
const callLoadFileInput = (
actor: Actor,
input: unknown,
options?: Parameters<typeof loadFileInput>[4],
) =>
withActor(actor, () =>
loadFileInput(
server.stores,
server.services.fs,
actor,
input,
options,
),
);
// ── inferFilenameFromUrlOrPath ─────────────────────────────────────
describe('inferFilenameFromUrlOrPath', () => {
it('returns the basename of a URL pathname', () => {
expect(inferFilenameFromUrlOrPath('https://cdn.test/a/b/photo.png')).toBe(
'photo.png',
);
});
it('returns the basename of a posix-style path string', () => {
expect(inferFilenameFromUrlOrPath('/alice/Music/song.mp3')).toBe(
'song.mp3',
);
});
it('falls back to the supplied default when there is no basename', () => {
// Empty string has no URL form and no posix basename, so the
// explicit fallback wins.
expect(inferFilenameFromUrlOrPath('', 'fallback-name')).toBe(
'fallback-name',
);
});
it('uses the literal `input` as fallback when no override is passed', () => {
expect(inferFilenameFromUrlOrPath('')).toBe('input');
});
it('handles bare filenames (no slashes, not a URL)', () => {
expect(inferFilenameFromUrlOrPath('plain.txt')).toBe('plain.txt');
});
});
// ── loadFileInput — argument validation ─────────────────────────────
describe('loadFileInput validation', () => {
it('throws 400 when input is empty/falsy', async () => {
const { actor } = await makeUser();
await expect(callLoadFileInput(actor, undefined)).rejects.toMatchObject(
{ statusCode: 400 },
);
await expect(callLoadFileInput(actor, null)).rejects.toMatchObject({
statusCode: 400,
});
await expect(callLoadFileInput(actor, '')).rejects.toMatchObject({
statusCode: 400,
});
});
it('throws 401 when actor.user.id is not a finite number', async () => {
// Missing actor entirely
await expect(
loadFileInput(
server.stores,
server.services.fs,
undefined as unknown as Actor,
'data:text/plain,hi',
),
).rejects.toMatchObject({ statusCode: 401 });
// Actor with no user
await expect(
loadFileInput(
server.stores,
server.services.fs,
{} as Actor,
'data:text/plain,hi',
),
).rejects.toMatchObject({ statusCode: 401 });
// Actor with non-numeric user.id
await expect(
loadFileInput(
server.stores,
server.services.fs,
{ user: { id: 'nope' } } as unknown as Actor,
'data:text/plain,hi',
),
).rejects.toMatchObject({ statusCode: 401 });
});
});
// ── loadFileInput — data URL path ───────────────────────────────────
describe('loadFileInput data URL', () => {
it('decodes a base64 data URL and reports the declared MIME', async () => {
const { actor } = await makeUser();
const payload = Buffer.from('hello world');
const dataUrl = `data:text/plain;base64,${payload.toString('base64')}`;
const result = await callLoadFileInput(actor, dataUrl);
expect(result.buffer.equals(payload)).toBe(true);
expect(result.mimeType).toBe('text/plain');
expect(result.fsEntry).toBeNull();
// Filename derives from the MIME subtype.
expect(result.filename).toBe('input.plain');
});
it('decodes a non-base64 (URL-encoded) data URL', async () => {
const { actor } = await makeUser();
// Plain (no `;base64`) → URL-decoded payload.
const result = await callLoadFileInput(
actor,
'data:text/plain,hello%20world',
);
expect(result.buffer.toString('utf8')).toBe('hello world');
expect(result.mimeType).toBe('text/plain');
});
it('infers MIME-derived filename for compound types like svg+xml', async () => {
const { actor } = await makeUser();
const result = await callLoadFileInput(
actor,
'data:image/svg+xml;base64,PHN2Zy8+',
);
// `image/svg+xml` → input.svg (subtype, before the `+`).
expect(result.filename).toBe('input.svg');
expect(result.mimeType).toBe('image/svg+xml');
});
// Audio upstreams pick the decoder from the extension; `input.mpeg` is
// rejected as corrupt.
it.each([
['audio/mpeg', 'input.mp3'],
['audio/x-wav', 'input.wav'],
['audio/wave', 'input.wav'],
['audio/x-m4a', 'input.m4a'],
['audio/ogg', 'input.ogg'],
])('names a %s data URL %s', async (mime, filename) => {
const { actor } = await makeUser();
const result = await callLoadFileInput(
actor,
`data:${mime};base64,QUJD`,
);
expect(result.filename).toBe(filename);
expect(result.mimeType).toBe(mime);
});
it('defaults MIME to application/octet-stream when omitted', async () => {
const { actor } = await makeUser();
const result = await callLoadFileInput(actor, 'data:;base64,QUJD');
expect(result.mimeType).toBe('application/octet-stream');
expect(result.buffer.toString('utf8')).toBe('ABC');
});
it('throws 400 on a malformed data URL', async () => {
const { actor } = await makeUser();
// Missing comma → DATA_URL_PATTERN.exec returns null.
await expect(
callLoadFileInput(actor, 'data:not-a-url'),
).rejects.toMatchObject({ statusCode: 400 });
});
it('rejects an oversize data URL with 413 + input_too_large', async () => {
const { actor } = await makeUser();
const payload = Buffer.alloc(64);
const dataUrl = `data:application/octet-stream;base64,${payload.toString('base64')}`;
await expect(
callLoadFileInput(actor, dataUrl, { maxBytes: 32 }),
).rejects.toMatchObject({
statusCode: 413,
legacyCode: 'input_too_large',
});
});
it('accepts a data URL exactly at the maxBytes threshold', async () => {
const { actor } = await makeUser();
const payload = Buffer.alloc(16, 0x41); // 16 bytes of 'A'
const dataUrl = `data:application/octet-stream;base64,${payload.toString('base64')}`;
const result = await callLoadFileInput(actor, dataUrl, {
maxBytes: 16,
});
expect(result.buffer.byteLength).toBe(16);
});
});
// ── loadFileInput — FS path ─────────────────────────────────────────
describe('loadFileInput FS path', () => {
it('reads bytes back through a real fsentry written via FSService', async () => {
const { actor, userId } = await makeUser();
const username = actor.user!.username!;
const path = `/${username}/Documents/sample.txt`;
const body = Buffer.from('hello from sql + s3');
const entry = await withActor(actor, () =>
writeFile(userId, path, body, 'text/plain'),
);
const result = await callLoadFileInput(actor, path);
expect(result.buffer.equals(body)).toBe(true);
expect(result.filename).toBe('sample.txt');
// FSService stamps the contentType into S3 metadata; loadFileInput
// returns it on the way out.
expect(result.mimeType).toBe('text/plain');
expect(result.fsEntry?.uuid).toBe(entry.uuid);
expect(result.fsEntry?.path).toBe(path);
});
it('returns empty bytes for an empty file with no backing S3 object', async () => {
// Files created via touch have size 0 and a null bucket — no S3
// object exists. loadFileInput must return empty content rather than
// throwing NoSuchKey from getObjectStream.
const { actor, userId } = await makeUser();
const username = actor.user!.username!;
const path = `/${username}/Documents/empty.txt`;
const entry = await withActor(actor, () =>
server.services.fs.touch(userId, { path }),
);
const result = await callLoadFileInput(actor, path);
expect(result.buffer.byteLength).toBe(0);
expect(result.fsEntry?.uuid).toBe(entry.uuid);
});
it('also accepts a `{ uuid }` object reference', async () => {
const { actor, userId } = await makeUser();
const username = actor.user!.username!;
const path = `/${username}/Documents/by-uuid.bin`;
const body = Buffer.from([0x01, 0x02, 0x03, 0x04]);
const entry = await withActor(actor, () =>
writeFile(userId, path, body),
);
const result = await callLoadFileInput(actor, { uuid: entry.uuid });
expect(result.buffer.equals(body)).toBe(true);
expect(result.fsEntry?.uuid).toBe(entry.uuid);
});
it('expands `~/...` paths against the actor home before resolving', async () => {
const { actor, userId } = await makeUser();
const username = actor.user!.username!;
const body = Buffer.from('tilde');
await withActor(actor, () =>
writeFile(userId, `/${username}/Documents/tilde.txt`, body),
);
const result = await callLoadFileInput(actor, '~/Documents/tilde.txt');
expect(result.buffer.equals(body)).toBe(true);
});
it('throws 404 when the fsentry cannot be resolved', async () => {
const { actor } = await makeUser();
await expect(
callLoadFileInput(actor, {
uuid: '00000000-0000-0000-0000-000000000000',
}),
).rejects.toMatchObject({ statusCode: 404 });
});
it('rejects directory entries with 400', async () => {
const { actor } = await makeUser();
const username = actor.user!.username!;
// The Documents folder is a real fsentry directory created by
// generateDefaultFsentries.
await expect(
callLoadFileInput(actor, `/${username}/Documents`),
).rejects.toMatchObject({ statusCode: 400 });
});
it("rejects when the FS access check denies the actor", async () => {
// Owner writes a file; intruder tries to read it. The real
// FSService.checkFSAccess walks the ACL and refuses.
const owner = await makeUser();
const intruder = await makeUser();
const ownerName = owner.actor.user!.username!;
const path = `/${ownerName}/Documents/private.txt`;
await withActor(owner.actor, () =>
writeFile(owner.userId, path, Buffer.from('owned')),
);
const err = await callLoadFileInput(intruder.actor, path).then(
() => null,
(e: unknown) => e,
);
const status = (err as { statusCode?: number } | null)?.statusCode;
// Access denied lands as 403; "can't see" lands as 404 — both
// are valid refusals from ACLService.getSafeAclError.
expect([403, 404]).toContain(status);
});
it('rejects up-front when contentLength exceeds maxBytes', async () => {
const { actor, userId } = await makeUser();
const username = actor.user!.username!;
const path = `/${username}/Documents/big.bin`;
const body = Buffer.alloc(2048, 0x42);
await withActor(actor, () =>
writeFile(userId, path, body, 'application/octet-stream'),
);
await expect(
callLoadFileInput(actor, path, { maxBytes: 64 }),
).rejects.toMatchObject({
statusCode: 413,
legacyCode: 'input_too_large',
});
});
it('rejects the load when storage fails partway through the read', async () => {
const { actor, userId } = await makeUser();
const path = `/${actor.user!.username!}/Documents/cut-off.bin`;
const entry = await withActor(actor, () =>
writeFile(userId, path, Buffer.alloc(64, 0x42)),
);
// The storage client is the boundary: one chunk, then the
// connection drops.
const client = server.clients.s3.get(
server.stores.s3Object.resolveRegion(entry.bucketRegion),
);
const send = client.send.bind(client);
const spy = vi.spyOn(client, 'send').mockImplementation(((
command: unknown,
) => {
if (!(command instanceof GetObjectCommand)) {
return send(command as never);
}
let reads = 0;
const body = new Readable({
read() {
reads += 1;
if (reads === 1) this.push(Buffer.alloc(16, 0x42));
else this.destroy(new Error('aborted'));
},
});
return Promise.resolve({ Body: body, ContentLength: 64 });
}) as never);
try {
await expect(callLoadFileInput(actor, path)).rejects.toThrow(
'aborted',
);
} finally {
spy.mockRestore();
}
});
});
// ── resolveFileInput ───────────────────────────────────────────────
describe('resolveFileInput', () => {
it('checks access to a Puter file without reading it, then reads it once on load', async () => {
const { actor, userId } = await makeUser();
const path = `/${actor.user!.username!}/Documents/lazy.txt`;
const body = Buffer.from('read me later');
const entry = await withActor(actor, () =>
writeFile(userId, path, body, 'text/plain'),
);
const s3Read = vi.spyOn(server.stores.s3Object, 'getObjectStream');
try {
const resolved = await withActor(actor, () =>
resolveFileInput(
server.stores,
server.services.fs,
actor,
path,
),
);
expect(resolved.filename).toBe('lazy.txt');
expect(resolved.fsEntry?.uuid).toBe(entry.uuid);
expect(s3Read).not.toHaveBeenCalled();
const [first, second] = await Promise.all([
resolved.load(),
resolved.load(),
]);
expect(first.buffer.equals(body)).toBe(true);
expect(second).toBe(first);
expect(s3Read).toHaveBeenCalledTimes(1);
} finally {
s3Read.mockRestore();
}
});
it('refuses a Puter file over maxBytes from its recorded size, before any read', async () => {
const { actor, userId } = await makeUser();
const path = `/${actor.user!.username!}/Documents/big.bin`;
await withActor(actor, () =>
writeFile(userId, path, Buffer.alloc(2048, 0x42)),
);
const s3Read = vi.spyOn(server.stores.s3Object, 'getObjectStream');
try {
await expect(
withActor(actor, () =>
resolveFileInput(
server.stores,
server.services.fs,
actor,
path,
{ maxBytes: 64 },
),
),
).rejects.toMatchObject({
statusCode: 413,
legacyCode: 'input_too_large',
});
expect(s3Read).not.toHaveBeenCalled();
} finally {
s3Read.mockRestore();
}
});
});
// ── loadFileInput web URL ──────────────────────────────────────────
/** A web stream that yields `chunks` one pull at a time, counting pulls. */
const chunkedBody = (chunks: Buffer[]) => {
const state = { pulls: 0, cancelled: false };
const body = new ReadableStream<Uint8Array>({
pull(controller) {
const next = chunks[state.pulls++];
if (next) controller.enqueue(new Uint8Array(next));
else controller.close();
},
cancel() {
state.cancelled = true;
},
});
return { body, state };
};
describe('loadFileInput web URL', () => {
it('returns the fetched bytes with the response content type', async () => {
const { actor } = await makeUser();
const { body } = chunkedBody([Buffer.from('hello '), Buffer.from('web')]);
secureFetchMock.mockResolvedValueOnce(
new Response(body, {
headers: { 'content-type': 'image/png; charset=binary' },
}),
);
const loaded = await callLoadFileInput(
actor,
'https://example.com/scan.png',
{ acceptWebInput: true, maxBytes: 64 },
);
expect(loaded.buffer.toString()).toBe('hello web');
expect(loaded.mimeType).toBe('image/png');
expect(loaded.filename).toBe('scan.png');
});
it('refuses a declared content-length over maxBytes without reading the body', async () => {
const { actor } = await makeUser();
const { body, state } = chunkedBody([Buffer.alloc(32)]);
secureFetchMock.mockResolvedValueOnce(
new Response(body, { headers: { 'content-length': '4096' } }),
);
await expect(
callLoadFileInput(actor, 'https://example.com/big.pdf', {
acceptWebInput: true,
maxBytes: 64,
}),
).rejects.toMatchObject({
statusCode: 413,
legacyCode: 'input_too_large',
});
expect(state.cancelled).toBe(true);
});
it('stops reading an undeclared-length body once it passes maxBytes', async () => {
const { actor } = await makeUser();
const chunks = Array.from({ length: 50 }, () => Buffer.alloc(40));
const { body, state } = chunkedBody(chunks);
secureFetchMock.mockResolvedValueOnce(new Response(body));
await expect(
callLoadFileInput(actor, 'https://example.com/stream', {
acceptWebInput: true,
maxBytes: 64,
}),
).rejects.toMatchObject({ statusCode: 413 });
expect(state.cancelled).toBe(true);
expect(state.pulls).toBeLessThan(chunks.length);
});
});