From e85cd9d53d8d7bfa286ba4a5966a32af7988119f Mon Sep 17 00:00:00 2001 From: Daniel Salazar Date: Sat, 25 Jul 2026 14:04:21 -0700 Subject: [PATCH] feat: readdir with depth (#3446) --- .../controllers/fs/FSController.test.ts | 242 +++++++++++++++++- src/backend/controllers/fs/FSController.ts | 155 ++++++++--- src/backend/controllers/fs/legacyFsHelpers.ts | 179 +++++++------ src/backend/services/fs/FSService.ts | 116 +++++---- src/backend/stores/fs/FSEntryStore.ts | 175 +++++++++---- .../modules/FileSystem/operations/readdir.js | 32 ++- .../FileSystem/utils/mapV2EntryToV1.js | 59 +++++ src/puter-js/test/fs.test.js | 184 +++++++++++++ src/puter-js/tests/api/suites/fs.suite.ts | 38 +++ 9 files changed, 969 insertions(+), 211 deletions(-) create mode 100644 src/puter-js/src/modules/FileSystem/utils/mapV2EntryToV1.js diff --git a/src/backend/controllers/fs/FSController.test.ts b/src/backend/controllers/fs/FSController.test.ts index dc08d2ac6..65ec1c30a 100644 --- a/src/backend/controllers/fs/FSController.test.ts +++ b/src/backend/controllers/fs/FSController.test.ts @@ -848,7 +848,11 @@ describe('FSController.readdirEntries', () => { makeRes().res, ), ), - ).rejects.toMatchObject({ statusCode: 400 }); + ).rejects.toMatchObject({ + statusCode: 400, + // Matches the legacy `/readdir` code the SDK moved off of. + legacyCode: 'dest_is_not_a_directory', + }); }); }); @@ -1678,6 +1682,242 @@ describe('FSController.readdirEntries pagination', () => { }); }); +// -- /readdir recursive (nested listing) -- + +describe('FSController.readdirEntries recursive', () => { + // Seed a nested tree under Documents/tree and return its base path. + // Relative depths: l1a/l1b = 1, l2a = 2, l3a = 3, l4a = 4. + const makeTree = async () => { + const { actor, userId } = await makeUser(); + const username = actor.user!.username!; + const base = `/${username}/Documents/tree`; + const dirs = [ + base, + `${base}/l1a`, + `${base}/l1b`, + `${base}/l1a/l2a`, + `${base}/l1a/l2a/l3a`, + `${base}/l1a/l2a/l3a/l4a`, + ]; + for (const path of dirs) { + await withActor(actor, () => + controller.mkdirEntry( + makeReq({ body: { path }, actor }), + makeRes().res, + ), + ); + } + return { actor, userId, base }; + }; + + const readdir = async ( + actor: Awaited>['actor'], + body: Record, + ) => { + const { res, captured } = makeRes(); + await withActor(actor, () => + controller.readdirEntries(makeReq({ body, actor }), res), + ); + return captured.body; + }; + + const rel = (base: string, items: Array<{ path: string }>) => + items.map((e) => e.path.slice(base.length + 1)).sort(); + + it('depth 1 returns only direct children (like a normal readdir)', async () => { + const { actor, base } = await makeTree(); + const page = (await readdir(actor, { + path: base, + recursive: true, + depth: 1, + })) as { items: Array<{ path: string }>; cursor?: string }; + expect(rel(base, page.items)).toEqual(['l1a', 'l1b']); + }); + + it('deeper levels appear as depth grows', async () => { + const { actor, base } = await makeTree(); + const d2 = (await readdir(actor, { + path: base, + recursive: true, + depth: 2, + })) as { items: Array<{ path: string }> }; + expect(rel(base, d2.items)).toEqual(['l1a', 'l1a/l2a', 'l1b']); + + const d3 = (await readdir(actor, { + path: base, + recursive: true, + depth: 3, + })) as { items: Array<{ path: string }> }; + expect(rel(base, d3.items)).toEqual([ + 'l1a', + 'l1a/l2a', + 'l1a/l2a/l3a', + 'l1b', + ]); + }); + + it('caps depth at 10 so a huge depth returns the whole subtree', async () => { + const { actor, base } = await makeTree(); + const page = (await readdir(actor, { + path: base, + recursive: true, + depth: 9999, + })) as { items: Array<{ path: string }> }; + expect(rel(base, page.items)).toEqual([ + 'l1a', + 'l1a/l2a', + 'l1a/l2a/l3a', + 'l1a/l2a/l3a/l4a', + 'l1b', + ]); + }); + + it('pages through the whole subtree with cursors, no dupes or gaps', async () => { + const { actor, base } = await makeTree(); + const seen: string[] = []; + let cursor: string | null | undefined = null; + do { + const page = (await readdir(actor, { + path: base, + recursive: true, + depth: 10, + limit: 2, + cursor, + })) as { items: Array<{ path: string }>; cursor?: string }; + expect(page.items.length).toBeLessThanOrEqual(2); + seen.push(...page.items.map((e) => e.path)); + cursor = page.cursor; + } while (cursor); + expect(rel(base, seen.map((path) => ({ path })))).toEqual([ + 'l1a', + 'l1a/l2a', + 'l1a/l2a/l3a', + 'l1a/l2a/l3a/l4a', + 'l1b', + ]); + }); + + it('counts the subtree with includeTotal', async () => { + const { actor, base } = await makeTree(); + const page = (await readdir(actor, { + path: base, + recursive: true, + depth: 2, + cursor: null, + includeTotal: true, + })) as { items: unknown[]; total?: number }; + expect(page.total).toBe(3); // l1a, l1b, l2a + }); + + it('enriches entries with type, thumbnail and associatedApp', async () => { + const { actor, base } = await makeTree(); + const page = (await readdir(actor, { + path: base, + recursive: true, + depth: 1, + })) as { + items: Array<{ + type?: unknown; + thumbnail?: unknown; + associatedApp?: unknown; + }>; + }; + for (const item of page.items) { + expect(item.type).toBe('folder'); + expect(item.thumbnail ?? null).toBeNull(); + expect('associatedApp' in item).toBe(true); + } + }); + + it('gives files a MIME type and an associatedApp field', async () => { + const { actor } = await makeUser(); + const username = actor.user!.username!; + const dir = `/${username}/Documents/files`; + await withActor(actor, () => + controller.mkdirEntry( + makeReq({ body: { path: dir }, actor }), + makeRes().res, + ), + ); + await withActor(actor, () => + controller.touchEntry( + makeReq({ body: { path: `${dir}/pic.png` }, actor }), + makeRes().res, + ), + ); + const page = (await readdir(actor, { + path: dir, + recursive: true, + depth: 1, + })) as { + items: Array<{ + name: string; + type?: unknown; + associatedApp?: unknown; + }>; + }; + const file = page.items.find((e) => e.name === 'pic.png')!; + expect(String(file.type)).toContain('image/png'); + expect('associatedApp' in file).toBe(true); + }); + + it('masks denials for app-under-user actors as a 404 (legacy parity)', async () => { + const { actor: userActor } = await makeUser(); + const username = userActor.user!.username!; + const appActor: Actor = { + ...userActor, + app: { uid: `app-readdir-${uuidv4()}` }, + }; + // The user's Documents is outside the app's AppData subtree, so the + // app can't list it. Legacy `/readdir` masks this as a 404 + // subject_does_not_exist rather than leaking a 403. + await expect( + withActor(appActor, () => + controller.readdirEntries( + makeReq({ + body: { path: `/${username}/Documents` }, + actor: appActor, + }), + makeRes().res, + ), + ), + ).rejects.toMatchObject({ + statusCode: 404, + legacyCode: 'subject_does_not_exist', + }); + }); + + it('rejects recursive listing at the root', async () => { + const { actor } = await makeUser(); + await expect( + withActor(actor, () => + controller.readdirEntries( + makeReq({ + body: { path: '/', recursive: true }, + actor, + }), + makeRes().res, + ), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + }); + + it('does not leak another users identically-named subtree', async () => { + const { actor, base } = await makeTree(); + // A second user with the same relative tree must not appear. + await makeTree(); + const page = (await readdir(actor, { + path: base, + recursive: true, + depth: 10, + })) as { items: Array<{ path: string }> }; + for (const item of page.items) { + expect(item.path.startsWith(`${base}/`)).toBe(true); + } + expect(page.items).toHaveLength(5); + }); +}); + // ── /touch additional branches ────────────────────────────────────── describe('FSController.touchEntry additional branches', () => { diff --git a/src/backend/controllers/fs/FSController.ts b/src/backend/controllers/fs/FSController.ts index 0f84c6ef0..c8b0d7ca8 100644 --- a/src/backend/controllers/fs/FSController.ts +++ b/src/backend/controllers/fs/FSController.ts @@ -3,18 +3,19 @@ * * This file is part of Puter. * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * Puter is free software: you can redistribute it and/or modify it under the + * terms of the GNU Affero General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more + * details. * * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . + * along with this program. If not, see + * [https://www.gnu.org/licenses/](https://www.gnu.org/licenses/). */ import Busboy from 'busboy'; @@ -60,7 +61,13 @@ import type { ThumbnailUploadPrepareItem, ThumbnailUploadPreparePayload, } from './types.js'; -import { toLegacyEntry } from './legacyFsHelpers.js'; +import { + toLegacyEntry, + loadLegacyAssociatedApps, + fsEntryMimeType, + signEntryThumbnail, + assertAccess as assertLegacyAccess, +} from './legacyFsHelpers.js'; class UploadProgressTracker implements UploadProgressTrackerLike { total = 0; progress = 0; @@ -89,6 +96,8 @@ class UploadProgressTracker implements UploadProgressTrackerLike { } const MAX_THUMBNAIL_BYTES = 2 * 1024 * 1024; +// Hard cap on how many levels below the target a recursive readdir descends. +const MAX_READDIR_DEPTH = 10; const DEFAULT_BATCH_ACL_CHECK_CONCURRENCY = 32; const DEFAULT_BATCH_WRITE_SIDE_EFFECT_CONCURRENCY = 8; @@ -883,7 +892,18 @@ export class FSController extends PuterController { Object.prototype.hasOwnProperty.call(body, 'cursor') || body.includeTotal === true; + // Undocumented: `recursive` lists descendants (prefix scan) up to + // `depth` levels below the target. Always paginated; sorts by path. + const recursive = this.#toBoolean(body.recursive) === true; + if (this.#isRootPathRef(body)) { + if (recursive) { + throw new HttpError( + 400, + 'recursive listing is not supported at the root', + { legacyCode: 'bad_request' }, + ); + } const { listRootEntries } = await import('../../services/fs/rootListing.js'); const rootChildren = await listRootEntries( @@ -901,9 +921,7 @@ export class FSController extends PuterController { child.suggestedApps = rootSuggestions[index] ?? []; } } - const rootItems = rootChildren.map((child) => - this.#toClientEntry(child), - ); + const rootItems = await this.#toReaddirEntries(rootChildren); if (paginated) { res.json({ items: rootItems, @@ -920,10 +938,19 @@ export class FSController extends PuterController { const parent = await this.#resolveEntryForRequest(body); if (!parent.isDir) { throw new HttpError(400, 'Target is not a directory', { - legacyCode: 'bad_request', + legacyCode: 'dest_is_not_a_directory', }); } - await this.#assertAccess(actor, parent.path, 'list'); + // Use the legacy access assertion so this endpoint stays behaviorally + // identical to the `/readdir` route the SDK moved off of — same error + // codes and the same app-actor 404 masking on denial. + await assertLegacyAccess( + this.services.acl, + this.services.fs, + actor, + parent.path, + 'list', + ); const limit = this.#toNumberOrUndefined(body.limit); const offset = this.#toNumberOrUndefined(body.offset); @@ -942,6 +969,41 @@ export class FSController extends PuterController { const sortOrder = (['asc', 'desc'] as const).find((v) => v === sortOrderRaw) ?? null; + if (recursive) { + const requestedDepth = this.#toNumberOrUndefined(body.depth); + const maxDepth = Math.min( + MAX_READDIR_DEPTH, + Math.max(1, Math.floor(requestedDepth ?? MAX_READDIR_DEPTH)), + ); + const page = await this.services.fs.listDirectoryTreePage( + parent.userId, + parent.path, + { + limit, + cursor: + typeof body.cursor === 'string' + ? body.cursor + : undefined, + maxDepth, + }, + ); + await this.#attachSuggestedApps(page.entries); + const total = + body.includeTotal === true + ? await this.services.fs.countDirectoryTree( + parent.userId, + parent.path, + maxDepth, + ) + : undefined; + res.json({ + items: await this.#toReaddirEntries(page.entries), + ...(page.cursor ? { cursor: page.cursor } : {}), + ...(total !== undefined ? { total } : {}), + }); + return; + } + if (paginated) { const page = await this.services.fs.listDirectoryPage(parent.uuid, { limit, @@ -956,7 +1018,7 @@ export class FSController extends PuterController { ? await this.services.fs.countDirectory(parent.uuid) : undefined; res.json({ - items: page.entries.map((child) => this.#toClientEntry(child)), + items: await this.#toReaddirEntries(page.entries), ...(page.cursor ? { cursor: page.cursor } : {}), ...(total !== undefined ? { total } : {}), }); @@ -970,7 +1032,37 @@ export class FSController extends PuterController { sortOrder, }); await this.#attachSuggestedApps(children); - res.json(children.map((child) => this.#toClientEntry(child))); + res.json(await this.#toReaddirEntries(children)); + } + + /** + * Shape readdir entries in the v2 (camelCase) response shape, enriched with + * the three fields the SDK cannot reconstruct on its own so it can rebuild + * the v1 shape: `type` (MIME), a signed `thumbnail`, and `associatedApp`. + */ + async #toReaddirEntries( + entries: FSEntry[], + ): Promise[]> { + const appsById = await loadLegacyAssociatedApps( + this.stores.app, + entries, + ); + return Promise.all( + entries.map(async (entry) => { + const shaped = this.#toClientEntry(entry); + shaped.type = fsEntryMimeType(entry); + shaped.thumbnail = await signEntryThumbnail( + this.clients.event, + entry.uuid, + entry.thumbnail, + ); + shaped.associatedApp = + entry.associatedAppId !== null + ? (appsById.get(entry.associatedAppId) ?? null) + : null; + return shaped; + }), + ); } async #attachSuggestedApps(entries: FSEntry[]): Promise { @@ -1343,12 +1435,12 @@ export class FSController extends PuterController { } /** - * Authorize creation of a new entry at `targetPath`. The standard rule - * is write on the parent, but we also accept write on the target itself - * — this lets an app create its own `//AppData/` folder - * (parent `AppData` is off-limits, but the target is the app's own - * subtree per ACLService's short-circuit) and lets recipients of a - * direct share on a not-yet-existent path materialize it. + * Authorize creation of a new entry at `targetPath`. The standard rule is + * write on the parent, but we also accept write on the target itself — this + * lets an app create its own `//AppData/` folder (parent + * `AppData` is off-limits, but the target is the app's own subtree per + * ACLService's short-circuit) and lets recipients of a direct share on a + * not-yet-existent path materialize it. */ async #assertCanCreate(actor: Actor, targetPath: string) { const parent = pathPosix.dirname(targetPath); @@ -1826,13 +1918,13 @@ export class FSController extends PuterController { /** * Bind `associatedAppId` onto the write input only when the actor is - * entitled to reference that app. `associatedAppId` is client-supplied - * and never trusted for authz, but it's echoed back in legacy FS - * responses — so an attacker could plant another tenant's private app id - * to confirm the row exists (an enumeration oracle) and harvest its - * metadata. Allow binding to public apps (their existence isn't secret) - * or to apps the actor owns; drop the association otherwise so the file - * simply carries no associated app. + * entitled to reference that app. `associatedAppId` is client-supplied and + * never trusted for authz, but it's echoed back in legacy FS responses — so + * an attacker could plant another tenant's private app id to confirm the + * row exists (an enumeration oracle) and harvest its metadata. Allow + * binding to public apps (their existence isn't secret) or to apps the + * actor owns; drop the association otherwise so the file simply carries no + * associated app. * * `#resolveWriteFileMetadata` has already copied the raw client value onto * `fileMetadata`, so a dropped association must be actively stripped — not @@ -1873,8 +1965,7 @@ export class FSController extends PuterController { // the ActorUser type. Access via the escape hatch until a proper // storage-quota mechanism is in place. const actorUser = req.actor?.user as - | Record - | undefined; + Record | undefined; const candidates = [ this.#toStorageCapacityCandidate(actorUser?.free_storage), diff --git a/src/backend/controllers/fs/legacyFsHelpers.ts b/src/backend/controllers/fs/legacyFsHelpers.ts index b123c82e9..b9de0cc3e 100644 --- a/src/backend/controllers/fs/legacyFsHelpers.ts +++ b/src/backend/controllers/fs/legacyFsHelpers.ts @@ -3,18 +3,19 @@ * * This file is part of Puter. * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * Puter is free software: you can redistribute it and/or modify it under the + * terms of the GNU Affero General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more + * details. * * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . + * along with this program. If not, see + * [https://www.gnu.org/licenses/](https://www.gnu.org/licenses/). */ import { posix as pathPosix } from 'node:path'; @@ -43,9 +44,8 @@ import type { IConfig } from '../../types.js'; /** * Shared helpers used by the legacy FS route shims (LegacyFSController). * - * Legacy clients speak snake_case and expect specific response shapes — - * these helpers encapsulate that translation so the route handlers stay - * terse. + * Legacy clients speak snake_case and expect specific response shapes — these + * helpers encapsulate that translation so the route handlers stay terse. */ // -- Body parsing ----------------------------------------------------- @@ -220,15 +220,15 @@ export async function assertAccess( } /** - * Authorize creation of a new entry at `targetPath`. The standard rule is - * write on the parent, but we also allow it when the actor has explicit - * write on the target itself — this covers an app creating its own - * `//AppData/` folder (parent `AppData` is off-limits, but - * the target is the app's own subtree per ACLService's short-circuit) and - * shares granted directly on a not-yet-created path. + * Authorize creation of a new entry at `targetPath`. The standard rule is write + * on the parent, but we also allow it when the actor has explicit write on the + * target itself — this covers an app creating its own + * `//AppData/` folder (parent `AppData` is off-limits, but the + * target is the app's own subtree per ACLService's short-circuit) and shares + * granted directly on a not-yet-created path. * - * On failure, delegates to `assertAccess` on the parent so the error - * shape stays identical to the previous parent-only check. + * On failure, delegates to `assertAccess` on the parent so the error shape + * stays identical to the previous parent-only check. */ export async function assertCanCreate( aclService: ACLService, @@ -272,20 +272,21 @@ const toIntBool = (v: unknown): number => (v ? 1 : 0); /** * Convert an AppStore-normalized app row into the v1 `associated_app` shape * embedded in legacy FS entries. Booleans round-trip back to integers (0/1) - * because the v1 wire contract emits them that way and existing clients key - * off it. Other columns pass through as-is — `metadata` is already parsed. + * because the v1 wire contract emits them that way and existing clients key off + * it. Other columns pass through as-is — `metadata` is already parsed. * - * Two redactions, because `associatedAppId` can name an app the actor does - * not own (the column is client-writable display metadata, never trusted for - * authz — see FSController) and may point cross-tenant: - * - Owner identifiers (`owner_user_id`, `app_owner`) are never emitted — - * they're internal user references with no client-side display use. - * - For private or protected apps, the direct hosting URL and launch - * capability flags (`index_url`, `godmode`, `maximize_on_start`, - * `background`, `metadata`) are dropped. This mirrors AppDriver, which - * withholds `index_url` from callers without read entitlement. Display - * fields (name, icon, title) still pass through so the GUI can label the - * file's associated app. + * Two redactions, because `associatedAppId` can name an app the actor does not + * own (the column is client-writable display metadata, never trusted for authz + * — see FSController) and may point cross-tenant: + * + * - Owner identifiers (`owner_user_id`, `app_owner`) are never emitted — they're + * internal user references with no client-side display use. + * - For private or protected apps, the direct hosting URL and launch capability + * flags (`index_url`, `godmode`, `maximize_on_start`, `background`, + * `metadata`) are dropped. This mirrors AppDriver, which withholds + * `index_url` from callers without read entitlement. Display fields (name, + * icon, title) still pass through so the GUI can label the file's associated + * app. */ function mapAppForLegacyAssociatedApp( app: Record, @@ -319,13 +320,15 @@ function mapAppForLegacyAssociatedApp( /** * Batch-load `associated_app` payloads for a set of entries. Dedupes app ids * across the input, hands them to `AppStore.getByIds` (one pipelined Redis - * MGET + a single `id IN (…)` query for any cache misses), and returns a map - * keyed by app id holding the v1-shaped embed. Callers pass the result to - * `toLegacyEntry` via `opts.appsById` so each entry hydrates without a - * second round-trip. + * MGET * - * Empty input short-circuits — readdir on a directory of plain files makes - * zero extra calls. + * - A single `id IN (…)` query for any cache misses), and returns a map keyed by + * app id holding the v1-shaped embed. Callers pass the result to + * `toLegacyEntry` via `opts.appsById` so each entry hydrates without a second + * round-trip. + * + * Empty input short-circuits — readdir on a directory of plain files makes zero + * extra calls. */ export async function loadLegacyAssociatedApps( appStore: AppRowLookup, @@ -348,12 +351,55 @@ export async function loadLegacyAssociatedApps( } /** - * Produce the snake_case entry shape legacy clients expect. If `thumbnail` - * is set, asks the thumbnail extension (via `thumbnail.read` event) to swap - * an S3 URL for a signed one. Pass `fsEntryStore`/`userStore` to hydrate - * `is_empty` (directories) and `owner` — both are required fields per - * the legacy stat contract but need extra DB lookups. Pass `appsById` - * (built via `loadLegacyAssociatedApps`) to populate `associated_app`. + * The v1 `type` field: a MIME content-type (e.g. "image/png; charset=utf-8") + * for files, or "folder" for directories. The GUI's icon lookup keys off + * `type.startsWith('image/')`, so a bare extension breaks icon selection. + */ +export function fsEntryMimeType(entry: { + isDir: boolean; + name: string; +}): string | null { + return entry.isDir ? 'folder' : contentTypeFromMime(entry.name) || null; +} + +/** + * Swap an S3 thumbnail key for a signed URL via the thumbnail extension + * (`thumbnail.read` event). Returns the input unchanged when there's no + * thumbnail or no event client; returns null if signing yields nothing. + */ +export async function signEntryThumbnail( + eventClient: EventClient | undefined, + uuid: string, + thumbnail: string | null, +): Promise { + if ( + typeof thumbnail !== 'string' || + thumbnail.length === 0 || + !eventClient + ) { + return thumbnail ?? null; + } + const thumbnailEntry = { uuid, thumbnail }; + try { + // emitAndWait — listener mutates `thumbnail` on the payload; plain + // `emit` is fire-and-forget and would drop the rewrite. + await eventClient.emitAndWait('thumbnail.read', thumbnailEntry, {}); + } catch { + // ignore — non-critical. + } + return typeof thumbnailEntry.thumbnail === 'string' && + thumbnailEntry.thumbnail.length > 0 + ? thumbnailEntry.thumbnail + : null; +} + +/** + * Produce the snake_case entry shape legacy clients expect. If `thumbnail` is + * set, asks the thumbnail extension (via `thumbnail.read` event) to swap an S3 + * URL for a signed one. Pass `fsEntryStore`/`userStore` to hydrate `is_empty` + * (directories) and `owner` — both are required fields per the legacy stat + * contract but need extra DB lookups. Pass `appsById` (built via + * `loadLegacyAssociatedApps`) to populate `associated_app`. */ export async function toLegacyEntry( eventClient: EventClient | undefined, @@ -367,13 +413,7 @@ export async function toLegacyEntry( } = {}, ): Promise> { const dirname = pathPosix.dirname(entry.path); - // v1 contract: `type` is a MIME content-type (e.g. "image/png; charset=utf-8") - // for files, or "folder" for directories. The GUI's icon lookup does - // `type.startsWith('image/')` etc., so a bare extension breaks every - // banner that falls through the name-extension ladder in item_icon.js. - const mimeType = entry.isDir - ? 'folder' - : contentTypeFromMime(entry.name) || null; + const mimeType = fsEntryMimeType(entry); const pathComponents = entry.path.split('/'); const appdata_app = @@ -444,28 +484,11 @@ export async function toLegacyEntry( } // Let the thumbnail extension swap an s3:// key for a signed URL. - if ( - typeof response.thumbnail === 'string' && - (response.thumbnail as string).length > 0 && - eventClient - ) { - const thumbnailEntry = { - uuid: entry.uuid, - thumbnail: response.thumbnail as string, - }; - try { - // emitAndWait — listener mutates `thumbnail` on the payload; - // plain `emit` is fire-and-forget and would drop the rewrite. - await eventClient.emitAndWait('thumbnail.read', thumbnailEntry, {}); - } catch { - // ignore — non-critical. - } - response.thumbnail = - typeof thumbnailEntry.thumbnail === 'string' && - thumbnailEntry.thumbnail.length > 0 - ? thumbnailEntry.thumbnail - : null; - } + response.thumbnail = await signEntryThumbnail( + eventClient, + entry.uuid, + response.thumbnail as string | null, + ); return response; } @@ -475,8 +498,8 @@ export { normalizeAbsolutePath }; // -- Signing --------------------------------------------------------- /** - * Pull the signing config off the app config. Throws if either value is - * missing — these are required for signed URL routes to function. + * Pull the signing config off the app config. Throws if either value is missing + * — these are required for signed URL routes to function. */ export function signingConfigFromAppConfig(config: IConfig): SigningConfig { const secret = config.url_signature_secret; @@ -498,9 +521,7 @@ export function signingConfigFromAppConfig(config: IConfig): SigningConfig { return { secret, apiBaseUrl }; } -/** - * Convenience wrapper: turn an FSEntry into a signed-file response object. - */ +/** Convenience wrapper: turn an FSEntry into a signed-file response object. */ export function signEntry( entry: { uuid: string; diff --git a/src/backend/services/fs/FSService.ts b/src/backend/services/fs/FSService.ts index e92b1d107..e6de631f6 100644 --- a/src/backend/services/fs/FSService.ts +++ b/src/backend/services/fs/FSService.ts @@ -3,18 +3,19 @@ * * This file is part of Puter. * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * Puter is free software: you can redistribute it and/or modify it under the + * terms of the GNU Affero General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more + * details. * * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . + * along with this program. If not, see + * [https://www.gnu.org/licenses/](https://www.gnu.org/licenses/). */ import { posix as pathPosix } from 'node:path'; @@ -124,10 +125,10 @@ export class FSService extends PuterService { } /** - * FS-domain permission rules. App/site/user registrations live in their - * own services (AppPermissionService, SubdomainPermissionService, - * AuthService). Splitting by domain keeps the dependency surface narrow: - * each service only pulls the stores it actually needs. + * FS-domain permission rules. App/site/user registrations live in their own + * services (AppPermissionService, SubdomainPermissionService, AuthService). + * Splitting by domain keeps the dependency surface narrow: each service + * only pulls the stores it actually needs. * * The path rewriter relies on `FSEntryStore.getEntryByPath`'s Redis cache * (60s TTL), which is invalidated on every rename/move/delete through the @@ -2760,12 +2761,36 @@ export class FSService extends PuterService { } /** - * Search by file name for a user. Linear-scan with LIKE — cheap for - * typical library sizes, revisit if we need full-text. + * Cursor-paginated nested listing: descendants of `path` up to `maxDepth` + * levels deep, ordered by path. Owner-scoped by `userId` + path prefix. + */ + async listDirectoryTreePage( + userId: number, + path: string, + options: { limit?: number; cursor?: string | null; maxDepth: number }, + ): Promise<{ entries: FSEntry[]; cursor?: string }> { + return this.stores.fsEntry.listDescendantsPage(userId, path, options); + } + + async countDirectoryTree( + userId: number, + path: string, + maxDepth: number, + ): Promise { + return this.stores.fsEntry.countDescendantsToDepth( + userId, + path, + maxDepth, + ); + } + + /** + * Search by file name for a user. Linear-scan with LIKE — cheap for typical + * library sizes, revisit if we need full-text. * * `pathScope` restricts results to entries at or under that path. - * App-under-user callers pass their AppData root so search can't leak - * paths the actor isn't allowed to read. + * App-under-user callers pass their AppData root so search can't leak paths + * the actor isn't allowed to read. */ async searchByName( userId: number, @@ -2793,8 +2818,8 @@ export class FSService extends PuterService { /** * Stream bytes of a file entry from S3. The returned stream is a Node - * Readable; caller pipes it into the HTTP response and emits metering - * once the stream ends. Honours HTTP Range when provided. + * Readable; caller pipes it into the HTTP response and emits metering once + * the stream ends. Honours HTTP Range when provided. * * Throws 400 if the entry isn't a file, 500 if the entry has no backing * bucket (should never happen for real files). @@ -2923,9 +2948,9 @@ export class FSService extends PuterService { } /** - * Resolve or create a parent directory for a given target path. Returns - * the parent entry. Throws 400 if the path has no parent (root) or 404 - * when parents are missing and create is disabled. + * Resolve or create a parent directory for a given target path. Returns the + * parent entry. Throws 400 if the path has no parent (root) or 404 when + * parents are missing and create is disabled. */ async #resolveOrCreateParent( userId: number, @@ -2951,9 +2976,10 @@ export class FSService extends PuterService { /** * Create a directory at `path`. Options: - * - overwrite: if a non-directory exists, remove it and create dir - * - dedupeName: if conflict, append ` (N)` - * - createMissingParents: create intermediate dirs + * + * - Overwrite: if a non-directory exists, remove it and create dir + * - DedupeName: if conflict, append ` (N)` + * - CreateMissingParents: create intermediate dirs * * Returns the created (or existing-on-dedupe-false-no-conflict) entry. */ @@ -3255,10 +3281,10 @@ export class FSService extends PuterService { /** * Hard-delete every FS entry owned by `userId`: S3 objects first, then - * every `fsentries` row. Used by account deletion. Paginates through - * files (5k at a time) so large users don't blow the heap, batches S3 - * deletes per bucket+region, and finishes with one bulk DELETE to - * sweep dirs/shortcuts/symlinks that don't have backing objects. + * every `fsentries` row. Used by account deletion. Paginates through files + * (5k at a time) so large users don't blow the heap, batches S3 deletes per + * bucket+region, and finishes with one bulk DELETE to sweep + * dirs/shortcuts/symlinks that don't have backing objects. * * Safe to call concurrently with other ops on the same user only in the * sense that orphaned S3 objects may linger if a write races us; the DB @@ -3369,13 +3395,12 @@ export class FSService extends PuterService { /** * Emit one of the lifecycle events that `extension.on('fs.…')` consumers * expect (cf-file-cache, future thumbnails-style extensions). Payload - * carries multiple aliases (`node` / `entry` / `uid`) so handlers using - * any existing calling convention just work. + * carries multiple aliases (`node` / `entry` / `uid`) so handlers using any + * existing calling convention just work. * - * Currently emitted: - * fs.create.{file,directory,shortcut,symlink} - * fs.write.file — overwrite of an existing file - * fs.rename — in-place name change (move emits fs.move.node separately) + * Currently emitted: fs.create.{file,directory,shortcut,symlink} + * fs.write.file — overwrite of an existing file fs.rename — in-place name + * change (move emits fs.move.node separately) * * Skipped intentionally: `fs.pending.*` (no real entry yet at signed-URL * issue time) and per-flavor `fs.move.file` (move already emits @@ -3404,8 +3429,8 @@ export class FSService extends PuterService { /** * Move an entry to a new parent (and optionally rename in the same op). - * Works for files and directories. Updates descendant paths when moving - * a directory. + * Works for files and directories. Updates descendant paths when moving a + * directory. */ async move( userId: number, @@ -3416,9 +3441,9 @@ export class FSService extends PuterService { overwrite?: boolean; dedupeName?: boolean; /** - * Optional metadata to overwrite on the moved entry. Callers use this - * for trash/restore: when moving into Trash the GUI stores - * `{ original_name, original_path, trashed_ts }` here so the restore + * Optional metadata to overwrite on the moved entry. Callers use + * this for trash/restore: when moving into Trash the GUI stores `{ + * original_name, original_path, trashed_ts }` here so the restore * path and trash listing can recover the pre-trash name. */ newMetadata?: Record | null; @@ -3516,10 +3541,10 @@ export class FSService extends PuterService { /** * Copy an entry to a new parent. For directories, walks descendants and - * issues S3 CopyObject + DB inserts. Thumbnail URLs on entries ride - * along in the DB column — the thumbnail extension is notified via - * `fs.copy.node` so it can duplicate the backing S3 object (otherwise - * deleting one copy would nuke the other's thumbnail). + * issues S3 CopyObject + DB inserts. Thumbnail URLs on entries ride along + * in the DB column — the thumbnail extension is notified via `fs.copy.node` + * so it can duplicate the backing S3 object (otherwise deleting one copy + * would nuke the other's thumbnail). */ async copy( userId: number, @@ -3782,7 +3807,8 @@ export class FSService extends PuterService { } /** - * This method checks if the specified actor has permission to access the entry provided. It will throw an error if the actor is not permitted + * This method checks if the specified actor has permission to access the + * entry provided. It will throw an error if the actor is not permitted */ async checkFSAccess( entry: FSEntry, diff --git a/src/backend/stores/fs/FSEntryStore.ts b/src/backend/stores/fs/FSEntryStore.ts index dcf2941c2..f2eee1ad0 100644 --- a/src/backend/stores/fs/FSEntryStore.ts +++ b/src/backend/stores/fs/FSEntryStore.ts @@ -3,18 +3,19 @@ * * This file is part of Puter. * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * Puter is free software: you can redistribute it and/or modify it under the + * terms of the GNU Affero General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more + * details. * * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . + * along with this program. If not, see + * [https://www.gnu.org/licenses/](https://www.gnu.org/licenses/). */ import { statfs } from 'node:fs/promises'; @@ -58,8 +59,8 @@ const DEFAULT_DB_CHUNK_CONCURRENCY = 4; /** * Store backing the `fsentries` table. Owns DB CRUD over filesystem entries, - * Redis caching keyed by uuid/path/id, and pending-upload-session state in - * the system KV store. Constructed by the store registry; depends on `kv`. + * Redis caching keyed by uuid/path/id, and pending-upload-session state in the + * system KV store. Constructed by the store registry; depends on `kv`. */ export class FSEntryStore extends PuterStore { declare protected stores: LayerInstances; @@ -971,12 +972,10 @@ export class FSEntryStore extends PuterStore { } /** - * Recursive-CTE lineage resolver. Returns the entry at `path`, or null - * if any segment is unresolvable. On success, backfills the `path` - * column on every row in the lineage (one UPDATE per row, gated on - * `path IS NULL OR path != expected` so we don't write the same value - * back on warm rows). - * + * Recursive-CTE lineage resolver. Returns the entry at `path`, or null if + * any segment is unresolvable. On success, backfills the `path` column on + * every row in the lineage (one UPDATE per row, gated on `path IS NULL OR + * path != expected` so we don't write the same value back on warm rows). */ async #resolveEntryByPathLineage( normalizedPath: string, @@ -1061,22 +1060,22 @@ export class FSEntryStore extends PuterStore { /** * Upward-walk variant of the lineage resolver. Used by uuid/id lookups - * where we already have the row but its `path` column is NULL — same - * legacy state the path-based resolver heals from the other direction. + * where we already have the row but its `path` column is NULL — same legacy + * state the path-based resolver heals from the other direction. * - * Walks `parent_uid` chains from the entry up to the home row - * (`parent_uid IS NULL`), then reuses `#backfillLineagePaths` to write - * the correct `path` on every ancestor + the entry itself. Returns the - * re-fetched target entry so the caller doesn't need a second query. + * Walks `parent_uid` chains from the entry up to the home row (`parent_uid + * IS NULL`), then reuses `#backfillLineagePaths` to write the correct + * `path` on every ancestor + the entry itself. Returns the re-fetched + * target entry so the caller doesn't need a second query. * - * Returns null if the chain is broken (an intermediate `parent_uid` - * doesn't resolve to a row) — those rows are unrecoverable from this - * direction and the caller should fall back to whatever degraded - * behaviour it had before. + * Returns null if the chain is broken (an intermediate `parent_uid` doesn't + * resolve to a row) — those rows are unrecoverable from this direction and + * the caller should fall back to whatever degraded behaviour it had + * before. * - * Same security posture as `#resolveEntryByPathLineage`: this method - * only resolves + heals data, it does not authorize access. Callers - * still gate via ACL on the returned entry. + * Same security posture as `#resolveEntryByPathLineage`: this method only + * resolves + heals data, it does not authorize access. Callers still gate + * via ACL on the returned entry. */ async #healEntryPathByLineageUp( targetUuid: string, @@ -1143,9 +1142,9 @@ export class FSEntryStore extends PuterStore { * Heal in place: for every entry in `entries` whose `path` column is NULL * (legacy rows that pre-date the path-cache write), run the lineage-up * resolver and replace the slot with the healed entry. Heals run in - * parallel — typical workloads have zero such rows so the loop is a - * no-op; rare bursts (a freshly migrated account) only pay one - * recursive-CTE round-trip per row instead of N sequential. + * parallel — typical workloads have zero such rows so the loop is a no-op; + * rare bursts (a freshly migrated account) only pay one recursive-CTE + * round-trip per row instead of N sequential. * * Mutates the array in place. */ @@ -1276,11 +1275,10 @@ export class FSEntryStore extends PuterStore { } /** - * Batched lookup by id. Dedupes input ids, reads cache via per-id GETs, - * and resolves remaining misses with a single - * `SELECT … WHERE id IN (…)` per chunk. Use this in place of - * `Promise.all(ids.map(getEntryById))` to avoid one connection per row - * on large id sets. + * Batched lookup by id. Dedupes input ids, reads cache via per-id GETs, and + * resolves remaining misses with a single `SELECT … WHERE id IN (…)` per + * chunk. Use this in place of `Promise.all(ids.map(getEntryById))` to avoid + * one connection per row on large id sets. * * Missing ids (no DB row) are simply absent from the returned map. */ @@ -2272,13 +2270,13 @@ export class FSEntryStore extends PuterStore { /** * Create a single non-file entry: directory, shortcut, or symlink. * - * Unlike `batchCreateEntries` (which is geared to S3-backed files), - * these rows carry no bucket metadata. The caller is responsible for - * parent/name conflict resolution — this method assumes the parent - * exists and the target name is free. + * Unlike `batchCreateEntries` (which is geared to S3-backed files), these + * rows carry no bucket metadata. The caller is responsible for parent/name + * conflict resolution — this method assumes the parent exists and the + * target name is free. * - * Returns the inserted entry with a refreshed row read. Throws 409 on - * a unique-key collision (caller should pre-check and dedupe). + * Returns the inserted entry with a refreshed row read. Throws 409 on a + * unique-key collision (caller should pre-check and dedupe). */ async createNonFileEntry(input: { userId: number; @@ -2374,8 +2372,8 @@ export class FSEntryStore extends PuterStore { } /** - * Update accessed/modified/created timestamps in place. Used by `touch` - * for entries that already exist. + * Update accessed/modified/created timestamps in place. Used by `touch` for + * entries that already exist. */ async touchEntryTimestamps( uuid: string, @@ -2480,8 +2478,7 @@ export class FSEntryStore extends PuterStore { } = {}, ): Promise<{ entries: FSEntry[]; cursor?: string }> { const payload = decodeCursor(options.cursor) as - | { v: unknown; id: number; s?: string; o?: string } - | undefined; + { v: unknown; id: number; s?: string; o?: string } | undefined; const requestedSort = options.sortBy ?? null; const requestedOrder = options.sortOrder ?? null; @@ -2630,6 +2627,88 @@ export class FSEntryStore extends PuterStore { return Number(rows[0]?.n ?? 0); } + // Slashes in a path — the depth marker. Paths are absolute and carry no + // trailing slash, so a direct child of `/u/foo` (2 slashes) is `/u/foo/x` + // (3 slashes): its depth relative to the prefix is 1. + #slashCount(value: string): number { + let count = 0; + for (let i = 0; i < value.length; i++) { + if (value[i] === '/') count++; + } + return count; + } + + /** + * Cursor-paginated descendants of a directory, limited to `maxDepth` levels + * below the prefix. Prefix + user_id scan (same index as + * `listDescendantsByPath`) with a portable slash-count depth filter. Keyset + * pagination on `path ASC` — path is unique per user, so no id tiebreaker. + */ + async listDescendantsPage( + userId: number, + pathPrefix: string, + options: { limit?: number; cursor?: string | null; maxDepth: number }, + ): Promise<{ entries: FSEntry[]; cursor?: string }> { + const normalizedPrefix = this.#normalizePath(pathPrefix); + if (normalizedPrefix === '/') { + throw new HttpError(400, 'Refusing to list descendants of root', { + legacyCode: 'bad_request', + }); + } + const likePattern = `${this.#escapeLikePattern(normalizedPrefix)}/%`; + const maxSlashes = + this.#slashCount(normalizedPrefix) + Math.max(1, options.maxDepth); + const limit = normalizeLimit(options.limit, { cap: 10_000 }) ?? 1000; + + const payload = decodeCursor(options.cursor) as + { p: string } | undefined; + const seek = payload ? 'AND path > ?' : ''; + const params: unknown[] = payload + ? [userId, likePattern, maxSlashes, payload.p, limit + 1] + : [userId, likePattern, maxSlashes, limit + 1]; + + const rows = (await this.clients.db.read( + `SELECT ${this.#selectFsentriesColumns()} + FROM fsentries + WHERE user_id = ? AND path LIKE ? ESCAPE '!' + AND (LENGTH(path) - LENGTH(REPLACE(path, '/', ''))) <= ? ${seek} + ORDER BY path ASC + LIMIT ?`, + params, + )) as unknown as FSEntryRow[]; + + const hasMore = rows.length > limit; + const pageRows = hasMore ? rows.slice(0, limit) : rows; + const entries = await this.#finalizeChildEntries(pageRows); + + let cursor: string | undefined; + if (hasMore) { + const last = pageRows[pageRows.length - 1]!; + cursor = encodeCursor({ p: last.path }); + } + + return { entries, ...(cursor ? { cursor } : {}) }; + } + + async countDescendantsToDepth( + userId: number, + pathPrefix: string, + maxDepth: number, + ): Promise { + const normalizedPrefix = this.#normalizePath(pathPrefix); + if (normalizedPrefix === '/') return 0; + const likePattern = `${this.#escapeLikePattern(normalizedPrefix)}/%`; + const maxSlashes = + this.#slashCount(normalizedPrefix) + Math.max(1, maxDepth); + const rows = (await this.clients.db.read( + `SELECT COUNT(*) AS n FROM fsentries + WHERE user_id = ? AND path LIKE ? ESCAPE '!' + AND (LENGTH(path) - LENGTH(REPLACE(path, '/', ''))) <= ?`, + [userId, likePattern, maxSlashes], + )) as unknown as { n: number | string }[]; + return Number(rows[0]?.n ?? 0); + } + // Sum of sizes under a path (inclusive). Files only — dirs have null size. // NOTE: linear scan under the path prefix index; optimize later if it // becomes hot (e.g., incremental size counters or materialized totals). diff --git a/src/puter-js/src/modules/FileSystem/operations/readdir.js b/src/puter-js/src/modules/FileSystem/operations/readdir.js index 93940ef80..3ac109812 100644 --- a/src/puter-js/src/modules/FileSystem/operations/readdir.js +++ b/src/puter-js/src/modules/FileSystem/operations/readdir.js @@ -1,6 +1,7 @@ import * as utils from '../../../lib/utils.js'; import { fetchAllPages, iteratePages } from '../../../lib/pagination.js'; import getAbsolutePathForApp from '../utils/getAbsolutePathForApp.js'; +import mapV2EntryToV1 from '../utils/mapV2EntryToV1.js'; // Track in-flight requests to avoid duplicate backend calls // Each entry stores: { promise, timestamp } @@ -27,17 +28,31 @@ const requestOnce = function (options, pageParams) { } } - // create xhr object - const xhr = utils.initXhr('/readdir', this.APIOrigin, undefined, 'post', 'text/plain;actually=json'); + // create xhr object. Backend serves readdir on the v2 `/fs/readdir` + // route, which returns camelCase entries; we normalize them to the v1 + // shape below so existing callers see an unchanged response. + const xhr = utils.initXhr('/fs/readdir', this.APIOrigin, undefined, 'post', 'text/plain;actually=json'); // set up event handlers for load and error events utils.setupXhrEventHandlers(xhr, undefined, undefined, (result) => { + // Normalize each v2 entry to the v1 shape, in place, so the bare + // array and the `{items, cursor?, total?}` envelope both return the + // legacy shape and the cache is populated with it. + let normalized; + if ( Array.isArray(result) ) { + normalized = result.map(mapV2EntryToV1); + } else if ( result && Array.isArray(result.items) ) { + normalized = { ...result, items: result.items.map(mapV2EntryToV1) }; + } else { + normalized = result; + } + // set each individual item's cache - const entries = Array.isArray(result) ? result : (result?.items ?? []); + const entries = Array.isArray(normalized) ? normalized : (normalized?.items ?? []); for ( const item of entries ) { puter._cache.set(`item:${ item.path}`, item); } - resolve(result); + resolve(normalized); }, reject); // Build request payload - support both path and uid parameters @@ -51,6 +66,8 @@ const requestOnce = function (options, pageParams) { if ( options.offset !== undefined ) payload.offset = options.offset; if ( options.sortBy !== undefined ) payload.sortBy = options.sortBy; if ( options.sortOrder !== undefined ) payload.sortOrder = options.sortOrder; + if ( options.recursive !== undefined ) payload.recursive = options.recursive; + if ( options.depth !== undefined ) payload.depth = options.depth; if ( pageParams ) { payload.cursor = pageParams.cursor ?? null; if ( pageParams.includeTotal !== undefined ) { @@ -125,9 +142,10 @@ const readdir = function (...args) { options.offset === undefined; // Generate cache key based on path. Only full listings are cached — - // pages and limit/offset-truncated results never are. + // pages and limit/offset-truncated results never are. Recursive + // listings are never cached (they'd collide with the direct listing). let cacheKey; - if ( options.path && unbound ) { + if ( options.path && unbound && ! options.recursive ) { cacheKey = `readdir:${ options.path}`; } @@ -154,6 +172,8 @@ const readdir = function (...args) { includeTotal: options.includeTotal, sortBy: options.sortBy, sortOrder: options.sortOrder, + recursive: options.recursive, + depth: options.depth, }); // Check if there's already an in-flight request for the same parameters diff --git a/src/puter-js/src/modules/FileSystem/utils/mapV2EntryToV1.js b/src/puter-js/src/modules/FileSystem/utils/mapV2EntryToV1.js new file mode 100644 index 000000000..bac5cb7a3 --- /dev/null +++ b/src/puter-js/src/modules/FileSystem/utils/mapV2EntryToV1.js @@ -0,0 +1,59 @@ +import path from '../../../lib/path.js'; + +/** + * Convert a v2 (camelCase) fsentry from `/fs/readdir` into the v1 snake_case + * shape existing callers and the GUI consume. The backend readdir response is + * enriched with the three fields the client can't reconstruct on its own + * (`type`, a signed `thumbnail`, and `associatedApp`); everything else is a + * rename or a trivial derivation here. Mirrors the backend's `toLegacyEntry`. + * + * @param {Record} entry + * @returns {Record} + */ +const mapV2EntryToV1 = (entry) => { + if ( ! entry || typeof entry !== 'object' ) return entry; + + const entryPath = typeof entry.path === 'string' ? entry.path : ''; + const dirname = entryPath ? path.dirname(entryPath) : entry.dirname; + const pathComponents = entryPath.split('/'); + const appdata_app = pathComponents[2] === 'AppData' + ? pathComponents[3] + : undefined; + const subdomains = Array.isArray(entry.subdomains) ? entry.subdomains : []; + + return { + id: entry.uuid, + uid: entry.uid ?? entry.uuid, + uuid: entry.uuid, + parent_id: entry.parentUid ?? null, + parent_uid: entry.parentUid ?? null, + path: entry.path, + dirname, + dirpath: dirname, + name: entry.name, + is_dir: Boolean(entry.isDir), + is_shortcut: entry.isShortcut ? 1 : 0, + shortcut_to: entry.shortcutTo ?? null, + is_symlink: entry.isSymlink ? 1 : 0, + symlink_path: entry.symlinkPath ?? null, + type: entry.type ?? null, + writable: true, + is_public: entry.isPublic ?? null, + thumbnail: entry.thumbnail ?? null, + immutable: Boolean(entry.immutable), + metadata: entry.metadata ?? null, + modified: entry.modified, + created: entry.created ?? null, + accessed: entry.accessed ?? null, + size: entry.size ?? null, + layout: entry.layout ?? null, + subdomains, + workers: Array.isArray(entry.workers) ? entry.workers : [], + has_website: entry.hasWebsite ?? subdomains.length > 0, + suggested_apps: entry.suggestedApps, + associated_app: entry.associatedApp ?? null, + appdata_app, + }; +}; + +export default mapV2EntryToV1; diff --git a/src/puter-js/test/fs.test.js b/src/puter-js/test/fs.test.js index 281744d68..2d966f315 100644 --- a/src/puter-js/test/fs.test.js +++ b/src/puter-js/test/fs.test.js @@ -141,6 +141,27 @@ const isApiPuterComOrigin = () => { } }; +// Build a known directory tree under `base` for the recursive-readdir tests: +// +// base/ +// top.txt (depth 1) +// a/ (depth 1) +// a1.txt (depth 2) +// b/ (depth 2) +// deep.txt (depth 3) +// +// 5 descendants total. Returns nothing — callers readdir `base` themselves. +const setupRecursiveTree = async (base) => { + await puter.fs.mkdir(base); + await puter.fs.write(`${base}/top.txt`, 'x'); + await puter.fs.write(`${base}/a/a1.txt`, 'x', { createMissingParents: true }); + await puter.fs.write(`${base}/a/b/deep.txt`, 'x', { createMissingParents: true }); +}; + +// Path relative to `base` for an entry whose absolute path contains `/base/…`. +// `base` is a unique randName, so splitting on `/${base}/` is unambiguous. +const relToBase = (base, absPath) => String(absPath).split(`/${base}/`)[1]; + window.fsTests = [ { name: "testFSWrite", @@ -953,4 +974,167 @@ window.fsTests = [ } } }, + { + name: "testFSReadDirRecursive", + description: "Test recursive readdir returns the whole subtree (descendants beyond direct children)", + test: async function() { + const base = puter.randName(); + try { + await setupRecursiveTree(base); + const result = await puter.fs.readdir({ path: base, recursive: true }); + assert(Array.isArray(result), "Recursive unbound readdir should return an array"); + const rel = result.map((e) => relToBase(base, e.path)).sort(); + assert( + JSON.stringify(rel) === JSON.stringify(['a', 'a/a1.txt', 'a/b', 'a/b/deep.txt', 'top.txt']), + "Recursive readdir returned the wrong subtree: " + JSON.stringify(rel), + ); + pass("testFSReadDirRecursive passed"); + } catch (error) { + fail("testFSReadDirRecursive failed:", error); + } finally { + try { await puter.fs.delete(base, { recursive: true }); } catch (e) {} + } + } + }, + { + name: "testFSReadDirRecursiveDepth", + description: "Test the recursive readdir `depth` option controls how many levels below the target are returned", + test: async function() { + const base = puter.randName(); + try { + await setupRecursiveTree(base); + + const d1 = await puter.fs.readdir({ path: base, recursive: true, depth: 1 }); + assert(d1.length === 2, "depth 1 should return only direct children, got " + d1.length); + + const d2 = await puter.fs.readdir({ path: base, recursive: true, depth: 2 }); + assert(d2.length === 4, "depth 2 should include grandchildren, got " + d2.length); + + const d3 = await puter.fs.readdir({ path: base, recursive: true, depth: 3 }); + assert(d3.length === 5, "depth 3 should include the whole tree, got " + d3.length); + + pass("testFSReadDirRecursiveDepth passed"); + } catch (error) { + fail("testFSReadDirRecursiveDepth failed:", error); + } finally { + try { await puter.fs.delete(base, { recursive: true }); } catch (e) {} + } + } + }, + { + name: "testFSReadDirRecursiveDepthCap", + description: "Test a very large recursive `depth` is capped server-side and simply returns the whole subtree", + test: async function() { + const base = puter.randName(); + try { + await setupRecursiveTree(base); + const result = await puter.fs.readdir({ path: base, recursive: true, depth: 9999 }); + assert(result.length === 5, "capped-depth recursive readdir should return the whole subtree, got " + result.length); + pass("testFSReadDirRecursiveDepthCap passed"); + } catch (error) { + fail("testFSReadDirRecursiveDepthCap failed:", error); + } finally { + try { await puter.fs.delete(base, { recursive: true }); } catch (e) {} + } + } + }, + { + name: "testFSReadDirRecursivePagination", + description: "Test recursive readdir pages through the whole subtree with a cursor, no duplicates or gaps", + test: async function() { + const base = puter.randName(); + try { + await setupRecursiveTree(base); + const seen = []; + let cursor = null; + let guard = 0; + do { + const page = await puter.fs.readdir({ path: base, recursive: true, depth: 10, limit: 2, cursor }); + assert(page.items.length <= 2, "each page should respect the limit"); + seen.push(...page.items.map((e) => relToBase(base, e.path))); + cursor = page.cursor; + assert(++guard < 20, "pagination did not terminate"); + } while (cursor); + assert( + JSON.stringify(seen.sort()) === JSON.stringify(['a', 'a/a1.txt', 'a/b', 'a/b/deep.txt', 'top.txt']), + "paged recursive readdir missed or duplicated entries: " + JSON.stringify(seen), + ); + pass("testFSReadDirRecursivePagination passed"); + } catch (error) { + fail("testFSReadDirRecursivePagination failed:", error); + } finally { + try { await puter.fs.delete(base, { recursive: true }); } catch (e) {} + } + } + }, + { + name: "testFSReadDirRecursiveIncludeTotal", + description: "Test recursive readdir with includeTotal reports the size of the subtree at the requested depth", + test: async function() { + const base = puter.randName(); + try { + await setupRecursiveTree(base); + const page = await puter.fs.readdir({ path: base, recursive: true, depth: 2, cursor: null, includeTotal: true }); + assert(page.total === 4, "includeTotal should count the depth-2 subtree (4), got " + page.total); + pass("testFSReadDirRecursiveIncludeTotal passed"); + } catch (error) { + fail("testFSReadDirRecursiveIncludeTotal failed:", error); + } finally { + try { await puter.fs.delete(base, { recursive: true }); } catch (e) {} + } + } + }, + { + name: "testFSReadDirRecursiveStream", + description: "Test recursive readdir with stream:true iterates the whole subtree page by page via for-await", + test: async function() { + const base = puter.randName(); + try { + await setupRecursiveTree(base); + const seen = []; + let pages = 0; + for await (const page of puter.fs.readdir({ path: base, recursive: true, depth: 10, limit: 2, stream: true })) { + pages++; + assert(page.items.length <= 2, "stream pages should respect the limit"); + seen.push(...page.items.map((e) => relToBase(base, e.path))); + } + assert(pages >= 2, "streaming a 5-entry tree with limit 2 should yield multiple pages"); + assert( + JSON.stringify(seen.sort()) === JSON.stringify(['a', 'a/a1.txt', 'a/b', 'a/b/deep.txt', 'top.txt']), + "streamed recursive readdir missed or duplicated entries: " + JSON.stringify(seen), + ); + pass("testFSReadDirRecursiveStream passed"); + } catch (error) { + fail("testFSReadDirRecursiveStream failed:", error); + } finally { + try { await puter.fs.delete(base, { recursive: true }); } catch (e) {} + } + } + }, + { + name: "testFSReadDirRecursiveV1Shape", + description: "Test recursive readdir entries keep the v1 shape (is_dir, path, name, MIME type) the GUI/apps expect", + test: async function() { + const base = puter.randName(); + try { + await setupRecursiveTree(base); + const result = await puter.fs.readdir({ path: base, recursive: true, depth: 10 }); + for (const entry of result) { + assert(typeof entry.is_dir === 'boolean', "entry.is_dir should be a boolean"); + assert(typeof entry.path === 'string' && entry.path.length > 0, "entry.path should be a non-empty string"); + assert(typeof entry.name === 'string' && entry.name.length > 0, "entry.name should be a non-empty string"); + assert('associated_app' in entry, "entry should carry associated_app"); + } + const dir = result.find((e) => e.name === 'a'); + const file = result.find((e) => e.name === 'deep.txt'); + assert(dir && dir.is_dir === true && dir.type === 'folder', "directory entry should have type 'folder'"); + assert(file && file.is_dir === false && String(file.type).includes('text/plain'), "text file should have a text/plain MIME type, got " + (file && file.type)); + pass("testFSReadDirRecursiveV1Shape passed"); + } catch (error) { + fail("testFSReadDirRecursiveV1Shape failed:", error); + } finally { + try { await puter.fs.delete(base, { recursive: true }); } catch (e) {} + } + } + }, ]; diff --git a/src/puter-js/tests/api/suites/fs.suite.ts b/src/puter-js/tests/api/suites/fs.suite.ts index 04011c9e3..bda46a6f3 100644 --- a/src/puter-js/tests/api/suites/fs.suite.ts +++ b/src/puter-js/tests/api/suites/fs.suite.ts @@ -247,6 +247,44 @@ export default suite('fs', { ); }, + 'readdir recursive lists a nested subtree in v1 shape': async (t) => { + const base = `${home(t)}/fs-suite-recursive`; + await t.puter.fs.write(`${base}/a/b/deep.txt`, 'x', { + createMissingParents: true, + }); + await t.puter.fs.write(`${base}/top.txt`, 'y'); + + // depth 1: only direct children + const shallow = (await t.puter.fs.readdir({ + path: base, + recursive: true, + depth: 1, + cursor: null, + })) as { items: Array<{ name: string; is_dir: unknown }> }; + const shallowNames = shallow.items.map((e) => e.name).sort(); + t.assert.deepEqual(shallowNames, ['a', 'top.txt']); + + // deeper: descendants appear, and paging terminates + const seen: string[] = []; + let cursor: string | null | undefined = null; + do { + const page = (await t.puter.fs.readdir({ + path: base, + recursive: true, + depth: 5, + limit: 2, + cursor, + })) as { + items: Array<{ path: string; is_dir: unknown }>; + cursor?: string; + }; + seen.push(...page.items.map((e) => e.path)); + cursor = page.cursor; + } while (cursor); + const rel = seen.map((p) => p.slice(base.length + 1)).sort(); + t.assert.deepEqual(rel, ['a', 'a/b', 'a/b/deep.txt', 'top.txt']); + }, + 'copy duplicates a file': async (t) => { const src = `${home(t)}/fs-suite-copy-src.txt`; const dstDir = `${home(t)}/fs-suite-copy-dst`;