From b6876c50bf9b9a5db22b97a122da5246eba803f1 Mon Sep 17 00:00:00 2001 From: Reynaldi Chernando <12949382+reynaldichernando@users.noreply.github.com> Date: Fri, 4 Sep 2026 21:31:33 +0700 Subject: [PATCH] PUT-1477 Add fs commands to cli (#3746) * Add fs commands to cli * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/cli/README.md | 78 +++ src/cli/bin/puter.js | 87 ++++ src/cli/package.json | 6 +- src/cli/src/commands/fs.js | 779 +++++++++++++++++++++++++++++ src/cli/src/commands/fs.test.js | 457 +++++++++++++++++ src/cli/src/commands/kv.js | 99 +--- src/cli/src/commands/site.js | 50 +- src/cli/src/lib/appTarget.js | 103 ++++ src/cli/src/lib/remotePath.js | 178 +++++++ src/cli/src/lib/remotePath.test.js | 116 +++++ src/cli/src/lib/transfer.js | 253 ++++++++++ src/cli/src/lib/transfer.test.js | 273 ++++++++++ src/cli/vitest.config.js | 8 + src/docs/src/cli.md | 146 +++++- 14 files changed, 2509 insertions(+), 124 deletions(-) create mode 100644 src/cli/src/commands/fs.js create mode 100644 src/cli/src/commands/fs.test.js create mode 100644 src/cli/src/lib/appTarget.js create mode 100644 src/cli/src/lib/remotePath.js create mode 100644 src/cli/src/lib/remotePath.test.js create mode 100644 src/cli/src/lib/transfer.js create mode 100644 src/cli/src/lib/transfer.test.js create mode 100644 src/cli/vitest.config.js diff --git a/src/cli/README.md b/src/cli/README.md index 53860027c..70cdc78f3 100644 --- a/src/cli/README.md +++ b/src/cli/README.md @@ -22,6 +22,84 @@ puter logout puter whoami ``` +## Files + +One-shot file operations against your Puter drive. A remote path carries a +`puter:` prefix and is absolute from your home directory, `-` is stdin or +stdout, and anything else is local — so the direction of a transfer is read off +the operands and there is no `--upload` flag to remember. + +```sh +puter fs ls puter:/Desktop [-l] [--json] +puter fs cat puter:/notes.txt +puter fs cp [-r] [-n] [--concurrency n] [--dry-run] +puter fs mv puter:/a puter:/b +puter fs rm puter:/old [-r] [-y] [--dry-run] +puter fs mkdir puter:/logs [-p] +puter fs stat puter:/notes.txt [--json] +``` + +`cp` picks its implementation from the pair of operands: + +| From | To | What happens | +| ---------- | ---------- | -------------------------------------------- | +| local | `puter:/…` | upload | +| `puter:/…` | local | download | +| `puter:/…` | `puter:/…` | copied server-side, never through the client | +| `-` | `puter:/…` | stdin is written to the file | +| `puter:/…` | `-` | raw bytes to stdout | + +Two local paths — or two of anything else — is an error rather than a guess. +`mv` is remote-to-remote only for now: deleting local files after a failed +upload is not a first impression worth making. + +### Piping + +Status, progress and prompts go to stderr and data goes to stdout, so a +listing feeds straight back in: + +```sh +puter fs ls puter:/logs | xargs -n1 puter fs cat +``` + +A terminal gets the human view — bare names, aligned `-l` columns, a progress +spinner. A pipe gets `puter:`-prefixed paths, one per line, and `--json` gives +the whole entry. + +### App storage + +`--app` resolves `puter:/` against one app's storage directory +(`~/AppData/`) instead of your home directory. It takes the same +identifiers as `puter kv connect`: an app name, a worker name or URL, or a uid. + +```console +$ puter fs ls --app notes puter:/ +puter:/settings.json +$ puter fs rm -r --app notes puter:/cache +rm -r puter:/cache → ~/AppData/app-1f2e3d4c…/cache (412 entries) [notes (app-1f2e3d4c…)] +``` + +It is a flag and nothing else — no environment variable, no saved default. +Because it changes what an absolute path means, the rebasing has to be visible +in the command itself, so everything that changes or removes files echoes the +path it resolved to. Paths are clamped to that root: `puter:/../../Documents` +is an error, not an escape. + +### Removing things + +- `rm` needs `-r` for a directory, and prints the entry count before it goes. +- `-r` prompts on a terminal and requires `-y` anywhere else. +- Bare `puter:/` is refused — no one-liner should empty a whole drive or app store. +- Anything recursive takes `--dry-run`. + +### Transfers + +Bulk copies move 8 files at a time (`--concurrency`, 1–32) and retry what +failed for a reason that might not repeat. A partial failure doesn't abort the +run: the failures are named at the end and the exit code is non-zero, so +uploading 8,000 files doesn't start over because number 3,000 hiccuped. `-n` +skips what already exists; without it `cp` overwrites, like `cp`. + ## Sites Deploy a static directory to a `*.puter.site` subdomain, then list, inspect, or diff --git a/src/cli/bin/puter.js b/src/cli/bin/puter.js index 4afcaeea5..703498a4d 100755 --- a/src/cli/bin/puter.js +++ b/src/cli/bin/puter.js @@ -20,6 +20,15 @@ import { } from '../src/commands/worker.js'; import { appList, appGet } from '../src/commands/app.js'; import { kvConnect } from '../src/commands/kv.js'; +import { + fsLs, + fsCat, + fsCp, + fsMv, + fsRm, + fsMkdir, + fsStat, +} from '../src/commands/fs.js'; // The Puter.js SDK emits duplicate "stray" rejections for failed API calls in // addition to rejecting the promise we await. We already route the awaited @@ -142,4 +151,82 @@ kv .argument('', 'app name, worker name or URL, or app uid') .action(action(kvConnect)); +// --- fs --------------------------------------------------------------------- + +const fsCmd = program + .command('fs') + .description('Work with files on your Puter drive (remote paths are puter:/...)'); + +// `--app` rebases puter:/ onto one app's storage directory. Flag only: because +// it changes what an absolute path means, it has to be visible in the command +// itself — an invisible default turns `rm -r puter:/dist` into a command whose +// target you can't work out by reading it. +const withApp = (cmd) => + cmd.option( + '--app ', + "resolve puter:/ against an app's storage (app name, uid, worker name or URL)", + ); + +withApp( + fsCmd + .command('ls') + .description('List a remote directory') + .argument('', 'remote path (puter:/...)') + .option('-l, --long', 'show type, size and modification time') + .option('--json', 'emit the full entries as JSON'), +).action(action(fsLs)); + +withApp( + fsCmd + .command('cat') + .description("Write a remote file's contents to stdout") + .argument('', 'remote path (puter:/...)'), +).action(action(fsCat)); + +withApp( + fsCmd + .command('cp') + .description('Copy between local and remote paths, or within the drive') + .argument('', "local path, puter:/... or '-' for stdin") + .argument('', "local path, puter:/... or '-' for stdout") + .option('-r, --recursive', 'copy directories') + .option('-n, --no-clobber', 'skip files that already exist') + .option('--concurrency ', 'parallel transfers (1-32, default 8)') + .option('--dry-run', 'list what would be copied'), +).action(action(fsCp)); + +withApp( + fsCmd + .command('mv') + .description('Move or rename within the drive') + .argument('', 'remote path (puter:/...)') + .argument('', 'remote path (puter:/...)'), +).action(action(fsMv)); + +withApp( + fsCmd + .command('rm') + .description('Delete a remote file or directory') + .argument('', 'remote path (puter:/...)') + .option('-r, --recursive', 'remove a directory and its contents') + .option('-y, --yes', 'skip confirmation') + .option('--dry-run', 'list what would be deleted'), +).action(action(fsRm)); + +withApp( + fsCmd + .command('mkdir') + .description('Create a remote directory') + .argument('', 'remote path (puter:/...)') + .option('-p, --parents', 'create missing parents, and succeed if it exists'), +).action(action(fsMkdir)); + +withApp( + fsCmd + .command('stat') + .description('Show details for one remote file or directory') + .argument('', 'remote path (puter:/...)') + .option('--json', 'emit the full entry as JSON'), +).action(action(fsStat)); + program.parseAsync(process.argv); diff --git a/src/cli/package.json b/src/cli/package.json index 191559cac..1afaa0ee3 100644 --- a/src/cli/package.json +++ b/src/cli/package.json @@ -1,6 +1,6 @@ { "name": "@heyputer/cli", - "version": "0.3.0", + "version": "0.4.0", "description": "Puter CLI - developer tooling from your terminal.", "license": "MIT", "author": "Puter Technologies Inc.", @@ -19,11 +19,11 @@ }, "scripts": { "start": "node ./bin/puter.js", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "vitest run" }, "dependencies": { "@clack/prompts": "^0.7.0", - "@heyputer/puter.js": "^2.5.1", + "@heyputer/puter.js": "^2.6.2", "chalk": "^5.3.0", "commander": "^12.1.0", "conf": "^13.0.0" diff --git a/src/cli/src/commands/fs.js b/src/cli/src/commands/fs.js new file mode 100644 index 000000000..f382ef6d3 --- /dev/null +++ b/src/cli/src/commands/fs.js @@ -0,0 +1,779 @@ +// `puter fs` — one-shot file operations against Puter's cloud drive. +// +// Every path is either remote (`puter:/...`), stdin/stdout (`-`), or local, +// and the direction of a transfer is read off the operands rather than a +// flag. See ../lib/remotePath.js for the resolution rules and ../lib/ +// transfer.js for how bulk transfers are batched and retried. + +import fs from 'node:fs'; +import path from 'node:path'; +import * as clack from '@clack/prompts'; + +import { resolveTarget } from '../lib/appTarget.js'; +import { ensureClient } from '../lib/auth.js'; +import { canAnimate, canPrompt } from '../lib/env.js'; +import { CLIError, messageOf } from '../lib/errors.js'; +import { walk } from '../lib/fswalk.js'; +import { + appRoot, + assertMovable, + assertRemoteOperand, + copyDirection, + HOME_ROOT, + isRemote, + isRoot, + remoteBasename, + remoteDirname, + remoteJoin, + resolveRemote, + toOperand, +} from '../lib/remotePath.js'; +import { + DEFAULT_CONCURRENCY, + pool, + readRemote, + uploadFiles, + withRetry, + writeChunk, +} from '../lib/transfer.js'; +import * as ui from '../lib/ui.js'; + +// How many failures to name before summarizing the rest. +const MAX_REPORTED_FAILURES = 10; + +// --- errors ---------------------------------------------------------------- + +function wrap(err, message) { + if (err instanceof CLIError) return err; + return new CLIError(`${message}: ${messageOf(err)}`); +} + +const isMissing = (err) => + Number(err?.status) === 404 || + err?.code === 'not_found' || + err?.code === 'subject_does_not_exist' || + /not found|does not exist/i.test(messageOf(err)); + +// --- context --------------------------------------------------------------- + +/** + * The client plus the remote root every path in this invocation resolves + * against. `--app` rebases that root onto one app's storage directory; the + * label goes into the echo that destructive commands print, so the rebasing + * is visible exactly when it matters. + */ +async function connect(opts) { + const puter = await ensureClient(); + if (!opts.app) return { puter, opts, root: HOME_ROOT, label: null }; + + const { name, uid, byUid, kind } = await resolveTarget(puter, opts.app); + const label = byUid + ? uid + : `${kind === 'worker' ? 'worker ' : ''}${name} (${uid})`; + return { puter, opts, root: appRoot(uid), label }; +} + +// With `--app` an absolute path means something other than what it reads as, +// so anything that changes or removes files says what it resolved to. +function echoResolved(ctx, command, resolved) { + if (!ctx.label) return; + ui.status(`${command} → ${resolved} ${ui.dim(`[${ctx.label}]`)}`); +} + +// --- remote helpers -------------------------------------------------------- + +async function statOrNull(puter, remotePath) { + try { + return await puter.fs.stat(remotePath); + } catch (err) { + if (isMissing(err)) return null; + throw wrap(err, `Could not read '${remotePath}'`); + } +} + +async function statRemote(puter, remotePath, label) { + const info = await statOrNull(puter, remotePath); + if (!info) throw new CLIError(`No such file or directory '${label}'.`); + return info; +} + +// readdir returns the full listing as an array, or a `{items}` page when +// pagination params were passed. +async function listAll(puter, arg) { + const result = await puter.fs.readdir(arg); + return Array.isArray(result) ? result : (result?.items ?? []); +} + +// One page with a total beats pulling a whole tree just to count it; a +// backend that ignores `includeTotal` falls back to the full listing. +async function countEntries(puter, target) { + const page = await puter.fs.readdir({ + path: target, + recursive: true, + includeTotal: true, + limit: 1, + }); + if (!Array.isArray(page) && Number.isFinite(page?.total)) return page.total; + if (Array.isArray(page)) return page.length; + if (!page?.cursor) return page?.items?.length ?? 0; + return (await listAll(puter, { path: target, recursive: true })).length; +} + +// The paths of a directory's files, relative to it. Used by `-n` — one +// recursive listing beats a stat per file. +async function remoteRelativeFiles(puter, dir) { + const info = await statOrNull(puter, dir); + if (!info?.is_dir) return new Set(); + + const base = info.path; + const relatives = new Set(); + for (const entry of await listAll(puter, { path: dir, recursive: true })) { + if (entry.is_dir) continue; + const entryPath = typeof entry.path === 'string' ? entry.path : ''; + if (entryPath.startsWith(`${base}/`)) { + relatives.add(entryPath.slice(base.length + 1)); + } + } + return relatives; +} + +// mkdir -p on the destination root. upload() creates the directories its +// items sit in, but not the root they sit under. +async function ensureRemoteDir(puter, dir) { + try { + await puter.fs.mkdir(dir, { createMissingParents: true, dedupeName: false }); + } catch (err) { + // Already there is the common case; a real failure surfaces on upload. + ui.debug('mkdir note:', messageOf(err)); + } +} + +// --- local helpers --------------------------------------------------------- + +function statLocal(target, label) { + try { + return fs.statSync(target); + } catch { + throw new CLIError(`No such file or directory '${label}'.`); + } +} + +function isLocalDir(target) { + try { + return fs.statSync(target).isDirectory(); + } catch { + return false; + } +} + +async function readStdinBuffer() { + const chunks = []; + for await (const chunk of process.stdin) chunks.push(chunk); + return Buffer.concat(chunks); +} + +// --- formatting ------------------------------------------------------------ + +const SIZE_UNITS = ['B', 'KB', 'MB', 'GB', 'TB']; + +function formatSize(bytes) { + if (!Number.isFinite(bytes)) return '-'; + let value = bytes; + let unit = 0; + while (value >= 1024 && unit < SIZE_UNITS.length - 1) { + value /= 1024; + unit++; + } + const rounded = unit === 0 ? value : value < 10 ? value.toFixed(1) : Math.round(value); + return `${rounded} ${SIZE_UNITS[unit]}`; +} + +// fsentry timestamps are unix seconds. +function formatTime(seconds) { + if (!seconds) return '-'; + const date = new Date(Number(seconds) * 1000); + if (Number.isNaN(date.getTime())) return '-'; + const pad = (n) => String(n).padStart(2, '0'); + return ( + `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}` + + ` ${pad(date.getHours())}:${pad(date.getMinutes())}` + ); +} + +function parseConcurrency(value) { + if (value === undefined) return DEFAULT_CONCURRENCY; + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1 || parsed > 32) { + throw new CLIError( + `--concurrency must be a whole number from 1 to 32 (got '${value}').`, + ); + } + return parsed; +} + +// Partial failure in a bulk transfer doesn't abort the run: the failures are +// named here and the non-zero exit comes from the summary. +function reportFailures(failures, total, verb) { + if (failures.length === 0) return; + for (const failure of failures.slice(0, MAX_REPORTED_FAILURES)) { + ui.warn(`${failure.rel}: ${failure.message}`); + } + if (failures.length > MAX_REPORTED_FAILURES) { + ui.info(`… and ${failures.length - MAX_REPORTED_FAILURES} more.`); + } + throw new CLIError( + `${failures.length} of ${total} file(s) failed to ${verb}.`, + ); +} + +// --- ls -------------------------------------------------------------------- + +function printLongListing(entries) { + const rows = entries.map((entry) => ({ + type: entry.is_dir ? 'd' : '-', + size: entry.is_dir ? '-' : formatSize(Number(entry.size)), + modified: formatTime(entry.modified), + name: entry.is_dir ? `${entry.name}/` : entry.name, + })); + const widest = (key) => rows.reduce((max, row) => Math.max(max, row[key].length), 0); + const sizeWidth = widest('size'); + const timeWidth = widest('modified'); + + for (const row of rows) { + ui.out( + `${row.type} ${row.size.padStart(sizeWidth)} ` + + `${row.modified.padEnd(timeWidth)} ${row.name}`, + ); + } +} + +export async function fsLs(pathArg, opts) { + assertRemoteOperand(pathArg); + const ctx = await connect(opts); + const target = resolveRemote(pathArg, ctx.root); + + try { + if (opts.json || opts.long) { + const entries = await listAll(ctx.puter, target); + if (opts.json) ui.out(JSON.stringify(entries, null, 2)); + else printLongListing(entries); + return; + } + + // A terminal gets bare names; a pipe gets operands that feed straight + // back into another `puter fs` command. + const human = canAnimate(); + const emit = (entries) => { + for (const entry of entries) { + ui.out( + human + ? entry.is_dir + ? `${entry.name}/` + : entry.name + : toOperand(remoteJoin(target, entry.name), ctx.root), + ); + } + }; + + // Streamed page by page, so a large directory starts printing instead of + // buffering. An SDK without a streaming readdir answers the same call + // with the whole listing, so use that rather than asking twice. + const listing = ctx.puter.fs.readdir({ path: target, stream: true }); + if (typeof listing?.[Symbol.asyncIterator] === 'function') { + for await (const page of listing) emit(page.items ?? []); + return; + } + const result = await listing; + emit(Array.isArray(result) ? result : (result?.items ?? [])); + } catch (err) { + throw wrap(err, `Could not list '${pathArg}'`); + } +} + +// --- cat ------------------------------------------------------------------- + +async function catRemote(ctx, pathArg) { + const target = resolveRemote(pathArg, ctx.root); + const info = await statRemote(ctx.puter, target, pathArg); + if (info.is_dir) { + throw new CLIError(`'${pathArg}' is a directory.`, { + hint: `List it with: puter fs ls ${pathArg}`, + }); + } + + try { + for await (const chunk of readRemote(ctx.puter, target, Number(info.size))) { + await writeChunk(process.stdout, chunk); + } + } catch (err) { + throw wrap(err, `Could not read '${pathArg}'`); + } +} + +export async function fsCat(pathArg, opts) { + assertRemoteOperand(pathArg); + await catRemote(await connect(opts), pathArg); +} + +// --- cp -------------------------------------------------------------------- + +async function uploadLocal(ctx, sourceArg, destArg, concurrency) { + const { puter, opts } = ctx; + const source = path.resolve(sourceArg); + const localStat = statLocal(source, sourceArg); + const destination = resolveRemote(destArg, ctx.root); + const destInfo = await statOrNull(puter, destination); + + // Like cp: into the destination when it's an existing directory, otherwise + // the destination names the copy itself. + const intoDir = Boolean(destInfo?.is_dir); + + if (!localStat.isDirectory()) { + const target = intoDir + ? remoteJoin(destination, path.basename(source)) + : destination; + if (opts.noClobber && (intoDir ? await statOrNull(puter, target) : destInfo)) { + ui.info(`Skipped ${target} (exists).`); + return; + } + if (opts.dryRun) { + ui.out(`${sourceArg} → ${target}`); + ui.info('Dry run — nothing was copied.'); + return; + } + try { + await withRetry(() => + puter.fs.write(target, fs.readFileSync(source), { + overwrite: !opts.noClobber, + dedupeName: false, + createMissingParents: true, + }), + ); + } catch (err) { + throw wrap(err, `Could not copy '${sourceArg}'`); + } + ui.success(`Copied to ${target}.`); + return; + } + + if (!opts.recursive) { + throw new CLIError(`'${sourceArg}' is a directory — pass -r to copy it.`); + } + + const targetDir = intoDir + ? remoteJoin(destination, path.basename(source)) + : destination; + const files = walk(source); + if (files.length === 0) { + ui.warn(`'${sourceArg}' contains no files.`); + return; + } + + echoResolved(ctx, `cp -r ${sourceArg} ${destArg}`, targetDir); + + let planned = files; + if (opts.noClobber) { + const existing = await remoteRelativeFiles(puter, targetDir); + planned = files.filter((file) => !existing.has(file.rel)); + const skipped = files.length - planned.length; + if (skipped > 0) ui.info(`Skipping ${skipped} file(s) that already exist.`); + } + + if (opts.dryRun) { + for (const file of planned) ui.out(`${file.full} → ${remoteJoin(targetDir, file.rel)}`); + ui.info(`Dry run — ${planned.length} file(s) would be copied.`); + return; + } + if (planned.length === 0) return; + + await ensureRemoteDir(puter, targetDir); + + const spinner = ui.spinner(`Copying ${planned.length} file(s) to ${targetDir}…`); + const { uploaded, failures } = await uploadFiles(puter, { + files: planned, + destination: targetDir, + concurrency, + noClobber: Boolean(opts.noClobber), + onProgress: (done, total) => spinner.message(`Copied ${done}/${total} file(s)…`), + onRetry: (err, attempt) => ui.debug(`retry ${attempt}:`, messageOf(err)), + }); + spinner.stop(`Copied ${uploaded} of ${planned.length} file(s).`); + + reportFailures(failures, planned.length, 'copy'); +} + +async function downloadFile(puter, remotePath, size, target) { + fs.mkdirSync(path.dirname(target), { recursive: true }); + + // Write beside the target and rename, so a failed transfer never leaves a + // truncated file under the name the caller asked for. + const partial = `${target}.puter-partial`; + const handle = fs.openSync(partial, 'w'); + try { + for await (const chunk of readRemote(puter, remotePath, size)) { + fs.writeSync(handle, chunk); + } + } catch (err) { + fs.closeSync(handle); + try { + fs.unlinkSync(partial); + } catch { + // nothing to clean up + } + throw err; + } + fs.closeSync(handle); + fs.renameSync(partial, target); +} + +async function downloadRemote(ctx, sourceArg, destArg, concurrency) { + const { puter, opts } = ctx; + const source = resolveRemote(sourceArg, ctx.root); + const info = await statRemote(puter, source, sourceArg); + const destination = path.resolve(destArg); + const intoDir = isLocalDir(destination); + + if (!info.is_dir) { + const target = intoDir + ? path.join(destination, remoteBasename(source)) + : destination; + if (opts.noClobber && fs.existsSync(target)) { + ui.info(`Skipped ${target} (exists).`); + return; + } + if (opts.dryRun) { + ui.out(`${source} → ${target}`); + ui.info('Dry run — nothing was copied.'); + return; + } + try { + await downloadFile(puter, source, Number(info.size), target); + } catch (err) { + throw wrap(err, `Could not copy '${sourceArg}'`); + } + ui.success(`Copied to ${target}.`); + return; + } + + if (!opts.recursive) { + throw new CLIError(`'${sourceArg}' is a directory — pass -r to copy it.`); + } + + const targetDir = intoDir + ? path.join(destination, remoteBasename(source)) + : destination; + // Entry paths come back resolved (`/username/...`), so relative paths are + // taken against the source's own resolved path rather than the `~` form. + const base = info.path; + const files = (await listAll(puter, { path: source, recursive: true })) + .filter((entry) => !entry.is_dir && typeof entry.path === 'string') + .filter((entry) => entry.path.startsWith(`${base}/`)) + .map((entry) => ({ + remote: entry.path, + rel: entry.path.slice(base.length + 1), + size: Number(entry.size), + })); + + if (files.length === 0) { + ui.warn(`'${sourceArg}' contains no files.`); + return; + } + + let planned = files; + if (opts.noClobber) { + planned = files.filter((file) => !fs.existsSync(path.join(targetDir, file.rel))); + const skipped = files.length - planned.length; + if (skipped > 0) ui.info(`Skipping ${skipped} file(s) that already exist.`); + } + + if (opts.dryRun) { + for (const file of planned) ui.out(`${file.remote} → ${path.join(targetDir, file.rel)}`); + ui.info(`Dry run — ${planned.length} file(s) would be copied.`); + return; + } + if (planned.length === 0) return; + + const failures = []; + let done = 0; + const spinner = ui.spinner(`Copying ${planned.length} file(s) to ${targetDir}…`); + await pool(planned, concurrency, async (file) => { + try { + await downloadFile(puter, file.remote, file.size, path.join(targetDir, file.rel)); + } catch (err) { + failures.push({ rel: file.rel, message: messageOf(err) }); + } + spinner.message(`Copied ${++done}/${planned.length} file(s)…`); + }); + spinner.stop(`Copied ${planned.length - failures.length} of ${planned.length} file(s).`); + + reportFailures(failures, planned.length, 'copy'); +} + +async function copyWithinDrive(ctx, sourceArg, destArg) { + const { puter, opts } = ctx; + const source = resolveRemote(sourceArg, ctx.root); + const destination = resolveRemote(destArg, ctx.root); + const info = await statRemote(puter, source, sourceArg); + + if (info.is_dir && !opts.recursive) { + throw new CLIError(`'${sourceArg}' is a directory — pass -r to copy it.`); + } + + const destInfo = await statOrNull(puter, destination); + const intoDir = Boolean(destInfo?.is_dir); + const finalPath = intoDir + ? remoteJoin(destination, remoteBasename(source)) + : destination; + + if (opts.noClobber) { + const existing = intoDir ? await statOrNull(puter, finalPath) : destInfo; + if (existing) { + ui.info(`Skipped ${finalPath} (exists).`); + return; + } + } + + echoResolved(ctx, `cp ${sourceArg} ${destArg}`, finalPath); + if (opts.dryRun) { + ui.out(`${source} → ${finalPath}`); + ui.info('Dry run — nothing was copied.'); + return; + } + + // copy() copies *into* an existing directory; anything else names the copy + // itself, which means splitting the new name off the destination. + try { + await withRetry(() => + puter.fs.copy(source, intoDir ? destination : remoteDirname(destination), { + overwrite: true, + dedupeName: false, + ...(intoDir ? {} : { newName: remoteBasename(destination) }), + }), + ); + } catch (err) { + throw wrap(err, `Could not copy '${sourceArg}'`); + } + ui.success(`Copied to ${finalPath}.`); +} + +async function writeStdinTo(ctx, destArg) { + const { puter, opts } = ctx; + const destination = resolveRemote(destArg, ctx.root); + const destInfo = await statOrNull(puter, destination); + if (destInfo?.is_dir) { + throw new CLIError(`'${destArg}' is a directory — name the file to write.`); + } + if (opts.noClobber && destInfo) { + ui.info(`Skipped ${destination} (exists).`); + return; + } + + const data = await readStdinBuffer(); + echoResolved(ctx, `cp - ${destArg}`, destination); + if (opts.dryRun) { + ui.info(`Dry run — ${data.length} byte(s) would be written to ${destination}.`); + return; + } + + try { + await withRetry(() => + puter.fs.write(destination, data, { + overwrite: !opts.noClobber, + dedupeName: false, + createMissingParents: true, + }), + ); + } catch (err) { + throw wrap(err, `Could not write '${destArg}'`); + } + ui.success(`Wrote ${data.length} byte(s) to ${destination}.`); +} + +export async function fsCp(sourceArg, destArg, opts) { + if (destArg === undefined) { + throw new CLIError('cp needs a source and a destination.', { + hint: 'Usage: puter fs cp ', + }); + } + // Everything that doesn't need the network happens before authenticating, + // so an impossible pair or a missing local file fails immediately. + const direction = copyDirection(sourceArg, destArg); + for (const operand of [sourceArg, destArg]) { + if (isRemote(operand)) assertRemoteOperand(operand); + } + if (direction === 'upload') statLocal(path.resolve(sourceArg), sourceArg); + const concurrency = parseConcurrency(opts.concurrency); + const ctx = await connect(opts); + // commander reports the negatable `--no-clobber` as `clobber: false`. + ctx.opts = { ...opts, noClobber: opts.clobber === false }; + + switch (direction) { + case 'upload': + return uploadLocal(ctx, sourceArg, destArg, concurrency); + case 'download': + return downloadRemote(ctx, sourceArg, destArg, concurrency); + case 'remoteCopy': + return copyWithinDrive(ctx, sourceArg, destArg); + case 'writeStdin': + return writeStdinTo(ctx, destArg); + case 'readStdout': + return catRemote(ctx, sourceArg); + default: + throw new CLIError(`Unsupported copy: '${sourceArg}' → '${destArg}'.`); + } +} + +// --- mv -------------------------------------------------------------------- + +export async function fsMv(sourceArg, destArg, opts) { + if (destArg === undefined) { + throw new CLIError('mv needs a source and a destination.', { + hint: 'Usage: puter fs mv ', + }); + } + assertMovable(sourceArg, destArg); + assertRemoteOperand(sourceArg); + assertRemoteOperand(destArg); + const ctx = await connect(opts); + + const source = resolveRemote(sourceArg, ctx.root); + const destination = resolveRemote(destArg, ctx.root); + if (isRoot(source, ctx.root)) { + throw new CLIError(`Refusing to move ${sourceArg} — that is the root itself.`); + } + await statRemote(ctx.puter, source, sourceArg); + + echoResolved(ctx, `mv ${sourceArg} ${destArg}`, destination); + + // move() works out on its own whether the destination is a directory to + // move into or the item's new path. + try { + await withRetry(() => + ctx.puter.fs.move(source, destination, { + overwrite: true, + dedupeName: false, + }), + ); + } catch (err) { + throw wrap(err, `Could not move '${sourceArg}'`); + } + ui.success(`Moved to ${destination}.`); +} + +// --- rm -------------------------------------------------------------------- + +export async function fsRm(pathArg, opts) { + assertRemoteOperand(pathArg); + const ctx = await connect(opts); + const target = resolveRemote(pathArg, ctx.root); + + if (isRoot(target, ctx.root)) { + throw new CLIError( + `Refusing to delete ${pathArg} — that is ${ctx.label ? `all of ${ctx.label}` : 'the whole drive'}.`, + { hint: 'Name something inside it instead.' }, + ); + } + + const info = await statRemote(ctx.puter, target, pathArg); + if (info.is_dir && !opts.recursive) { + throw new CLIError( + `'${pathArg}' is a directory — pass -r to remove it and its contents.`, + ); + } + + if (opts.dryRun) { + if (info.is_dir) { + for (const entry of await listAll(ctx.puter, { path: target, recursive: true })) { + ui.out(entry.path); + } + } + ui.out(info.path ?? target); + ui.info('Dry run — nothing was deleted.'); + return; + } + + // What is about to go, in resolved terms, before anything goes. + const count = info.is_dir ? await countEntries(ctx.puter, target) : 0; + const what = info.is_dir + ? `${target} (${count} ${count === 1 ? 'entry' : 'entries'})` + : target; + if (opts.recursive || ctx.label) { + ui.status( + `rm ${opts.recursive ? '-r ' : ''}${pathArg} → ${what}` + + `${ctx.label ? ` ${ui.dim(`[${ctx.label}]`)}` : ''}`, + ); + } + + if (opts.recursive && !opts.yes) { + if (!canPrompt()) { + throw new CLIError('Refusing to delete recursively without confirmation.', { + hint: 'Pass --yes to confirm, or --dry-run to see what would go.', + }); + } + const go = await clack.confirm({ + message: `Delete ${target} and its ${count} entr${count === 1 ? 'y' : 'ies'}?`, + initialValue: false, + }); + if (clack.isCancel(go) || !go) throw new CLIError('Cancelled.'); + } + + // The SDK deletes recursively by default; a plain `rm` must say otherwise. + try { + await withRetry(() => + ctx.puter.fs.delete(target, { recursive: Boolean(opts.recursive) }), + ); + } catch (err) { + throw wrap(err, `Could not delete '${pathArg}'`); + } + ui.success(`Deleted ${target}.`); +} + +// --- mkdir ----------------------------------------------------------------- + +export async function fsMkdir(pathArg, opts) { + assertRemoteOperand(pathArg); + const ctx = await connect(opts); + const target = resolveRemote(pathArg, ctx.root); + if (isRoot(target, ctx.root)) { + throw new CLIError(`${pathArg} already exists.`); + } + + echoResolved(ctx, `mkdir ${pathArg}`, target); + + let item; + try { + item = await withRetry(() => + ctx.puter.fs.mkdir(target, { + createMissingParents: Boolean(opts.parents), + dedupeName: false, + }), + ); + } catch (err) { + throw wrap(err, `Could not create '${pathArg}'`); + } + ui.out(item?.path ?? target); +} + +// --- stat ------------------------------------------------------------------ + +export async function fsStat(pathArg, opts) { + assertRemoteOperand(pathArg); + const ctx = await connect(opts); + const target = resolveRemote(pathArg, ctx.root); + const info = await statRemote(ctx.puter, target, pathArg); + + if (opts.json) { + ui.out(JSON.stringify(info, null, 2)); + return; + } + + ui.out(`path: ${info.path ?? target}`); + ui.out(`type: ${info.is_dir ? 'directory' : (info.type ?? 'file')}`); + if (!info.is_dir) { + ui.out(`size: ${info.size ?? 0} (${formatSize(Number(info.size))})`); + } + ui.out(`modified: ${formatTime(info.modified)}`); + if (info.created) ui.out(`created: ${formatTime(info.created)}`); + if (info.uid) ui.out(`uid: ${info.uid}`); + if (info.is_public) ui.out('public: yes'); +} diff --git a/src/cli/src/commands/fs.test.js b/src/cli/src/commands/fs.test.js new file mode 100644 index 000000000..b90816f4f --- /dev/null +++ b/src/cli/src/commands/fs.test.js @@ -0,0 +1,457 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const state = vi.hoisted(() => ({ client: null })); + +vi.mock('../lib/auth.js', () => ({ + ensureClient: async () => state.client, +})); + +import { fsCat, fsCp, fsLs, fsMkdir, fsMv, fsRm, fsStat } from './fs.js'; + +const MODIFIED = 1756900000; + +// Entries are keyed by the `~/...` path a command resolves to, but carry the +// resolved `/u/...` path the backend actually returns. +function entry(remotePath, over = {}) { + const name = remotePath.split('/').pop(); + return { + path: `/u${remotePath.replace(/^~/, '')}`, + name, + is_dir: false, + size: 1024, + modified: MODIFIED, + uid: `uid-${name}`, + ...over, + }; +} + +const dir = (remotePath, over = {}) => + entry(remotePath, { is_dir: true, size: null, ...over }); + +const notFound = () => ({ + status: 404, + code: 'subject_does_not_exist', + message: 'not found', +}); + +function makeClient({ entries = {}, listings = {}, deep = {}, apps = {}, read } = {}) { + const calls = []; + const push = (name, ...args) => calls.push({ name, args }); + + async function* iterate(items) { + yield { items }; + } + + const client = { + fs: { + stat: async (arg) => { + const target = typeof arg === 'string' ? arg : arg.path; + push('stat', target); + if (!(target in entries)) throw notFound(); + return entries[target]; + }, + readdir: (arg) => { + const options = typeof arg === 'string' ? { path: arg } : arg; + push('readdir', options); + const items = options.recursive + ? (deep[options.path] ?? []) + : (listings[options.path] ?? []); + if (options.stream) return iterate(items); + if (options.includeTotal) { + return Promise.resolve({ + items: items.slice(0, options.limit ?? items.length), + total: items.length, + }); + } + return Promise.resolve(items); + }, + delete: async (target, options) => push('delete', target, options), + copy: async (source, destination, options) => + push('copy', source, destination, options), + move: async (source, destination, options) => + push('move', source, destination, options), + rename: async (...args) => push('rename', ...args), + mkdir: async (target, options) => { + push('mkdir', target, options); + return { path: `/u${target.replace(/^~/, '')}` }; + }, + write: async (target, data, options) => { + push('write', target, data.length, options); + return { path: target }; + }, + upload: async (items, destination, options) => { + push('upload', items.map((i) => i.finalPath), destination, options); + }, + read: read ?? (async () => new Blob([Buffer.from('hello')])), + }, + apps: { + list: async () => Object.values(apps), + get: async (name) => { + if (!apps[name]) throw notFound(); + return apps[name]; + }, + }, + workers: { + get: async () => null, + }, + }; + return { client, calls }; +} + +const callsOf = (calls, name) => calls.filter((call) => call.name === name); +const firstOf = (calls, name) => callsOf(calls, name)[0]; + +let out; +let err; + +beforeEach(() => { + out = []; + err = []; + vi.spyOn(console, 'log').mockImplementation((line = '') => out.push(String(line))); + vi.spyOn(console, 'error').mockImplementation((...args) => err.push(args.join(' '))); +}); + +afterEach(() => { + vi.restoreAllMocks(); + state.client = null; +}); + +describe('fs rm', () => { + it('deletes a file non-recursively — the SDK would recurse by default', async () => { + const { client, calls } = makeClient({ entries: { '~/a.txt': entry('~/a.txt') } }); + state.client = client; + + await fsRm('puter:/a.txt', {}); + + expect(firstOf(calls, 'delete').args).toEqual(['~/a.txt', { recursive: false }]); + }); + + it('refuses a directory without -r, and deletes nothing', async () => { + const { client, calls } = makeClient({ entries: { '~/dist': dir('~/dist') } }); + state.client = client; + + await expect(fsRm('puter:/dist', {})).rejects.toThrow(/is a directory — pass -r/); + expect(callsOf(calls, 'delete')).toHaveLength(0); + }); + + it('counts what is about to go before it goes', async () => { + const { client, calls } = makeClient({ + entries: { '~/dist': dir('~/dist') }, + deep: { '~/dist': [entry('~/dist/a.js'), entry('~/dist/b.js'), dir('~/dist/sub')] }, + }); + state.client = client; + + await fsRm('puter:/dist', { recursive: true, yes: true }); + + expect(err.join('\n')).toContain('rm -r puter:/dist → ~/dist (3 entries)'); + expect(firstOf(calls, 'delete').args).toEqual(['~/dist', { recursive: true }]); + }); + + it('refuses the root in both modes', async () => { + const { client, calls } = makeClient({ entries: { '~': dir('~') } }); + state.client = client; + + await expect(fsRm('puter:/', { recursive: true, yes: true })).rejects.toThrow( + /whole drive/, + ); + await expect( + fsRm('puter:/', { recursive: true, yes: true, app: 'app-1f2e' }), + ).rejects.toThrow(/all of app-1f2e/); + expect(callsOf(calls, 'delete')).toHaveLength(0); + }); + + it('will not delete recursively without confirmation when nobody can answer', async () => { + const { client, calls } = makeClient({ + entries: { '~/dist': dir('~/dist') }, + deep: { '~/dist': [entry('~/dist/a.js')] }, + }); + state.client = client; + + await expect(fsRm('puter:/dist', { recursive: true })).rejects.toThrow( + /without confirmation/, + ); + expect(callsOf(calls, 'delete')).toHaveLength(0); + }); + + it('lists instead of deleting on --dry-run', async () => { + const { client, calls } = makeClient({ + entries: { '~/dist': dir('~/dist') }, + deep: { '~/dist': [entry('~/dist/a.js')] }, + }); + state.client = client; + + await fsRm('puter:/dist', { recursive: true, dryRun: true }); + + expect(out).toEqual(['/u/dist/a.js', '/u/dist']); + expect(callsOf(calls, 'delete')).toHaveLength(0); + }); +}); + +describe('fs --app', () => { + it('rebases puter:/ onto the app store and says so', async () => { + const { client, calls } = makeClient({ entries: {} }); + state.client = client; + + await fsMkdir('puter:/dist', { app: 'app-1f2e', parents: true }); + + expect(firstOf(calls, 'mkdir').args).toEqual([ + '~/AppData/app-1f2e/dist', + { createMissingParents: true, dedupeName: false }, + ]); + expect(err.join('\n')).toContain('mkdir puter:/dist → ~/AppData/app-1f2e/dist'); + }); + + it('resolves an app name to its uid, the way kv connect does', async () => { + const { client, calls } = makeClient({ + apps: { notes: { name: 'notes', uid: 'app-99' } }, + }); + state.client = client; + + await fsMkdir('puter:/dist', { app: 'notes' }); + + expect(firstOf(calls, 'mkdir').args[0]).toBe('~/AppData/app-99/dist'); + expect(err.join('\n')).toContain('[notes (app-99)]'); + }); +}); + +describe('fs cp within the drive', () => { + it('copies into an existing directory under the same name', async () => { + const { client, calls } = makeClient({ + entries: { '~/a.txt': entry('~/a.txt'), '~/backup': dir('~/backup') }, + }); + state.client = client; + + await fsCp('puter:/a.txt', 'puter:/backup', {}); + + expect(firstOf(calls, 'copy').args).toEqual([ + '~/a.txt', + '~/backup', + { overwrite: true, dedupeName: false }, + ]); + }); + + it('splits the new name off a destination that does not exist yet', async () => { + const { client, calls } = makeClient({ entries: { '~/a.txt': entry('~/a.txt') } }); + state.client = client; + + await fsCp('puter:/a.txt', 'puter:/b.txt', {}); + + expect(firstOf(calls, 'copy').args).toEqual([ + '~/a.txt', + '~', + { overwrite: true, dedupeName: false, newName: 'b.txt' }, + ]); + }); + + it('skips an existing destination with -n', async () => { + const { client, calls } = makeClient({ + entries: { '~/a.txt': entry('~/a.txt'), '~/b.txt': entry('~/b.txt') }, + }); + state.client = client; + + // commander reports `--no-clobber` as clobber: false. + await fsCp('puter:/a.txt', 'puter:/b.txt', { clobber: false }); + + expect(callsOf(calls, 'copy')).toHaveLength(0); + expect(err.join('\n')).toContain('Skipped ~/b.txt (exists)'); + }); + + it('refuses a directory source without -r', async () => { + const { client, calls } = makeClient({ entries: { '~/dist': dir('~/dist') } }); + state.client = client; + + await expect(fsCp('puter:/dist', 'puter:/copy', {})).rejects.toThrow( + /is a directory — pass -r/, + ); + expect(callsOf(calls, 'copy')).toHaveLength(0); + }); +}); + +describe('fs cp across the boundary', () => { + const tempDirs = []; + + afterEach(() => { + while (tempDirs.length > 0) { + fs.rmSync(tempDirs.pop(), { recursive: true, force: true }); + } + }); + + function tempFile(name, contents) { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'puter-cli-fs-')); + tempDirs.push(base); + const full = path.join(base, name); + fs.writeFileSync(full, contents); + return full; + } + + it('writes a single local file to the remote path it names', async () => { + const { client, calls } = makeClient({ entries: {} }); + state.client = client; + const local = tempFile('notes.txt', 'hello'); + + await fsCp(local, 'puter:/notes.txt', {}); + + expect(firstOf(calls, 'write').args).toEqual([ + '~/notes.txt', + 5, + { overwrite: true, dedupeName: false, createMissingParents: true }, + ]); + }); + + it('uploads a local file into an existing remote directory', async () => { + const { client, calls } = makeClient({ entries: { '~/Desktop': dir('~/Desktop') } }); + state.client = client; + const local = tempFile('notes.txt', 'hello'); + + await fsCp(local, 'puter:/Desktop', {}); + + expect(firstOf(calls, 'write').args[0]).toBe('~/Desktop/notes.txt'); + }); + + it('writes stdout for a remote source and `-`', async () => { + const { client } = makeClient({ entries: { '~/a.txt': entry('~/a.txt', { size: 5 }) } }); + state.client = client; + const chunks = []; + vi.spyOn(process.stdout, 'write').mockImplementation((chunk, cb) => { + chunks.push(Buffer.from(chunk).toString()); + cb?.(); + return true; + }); + + await fsCp('puter:/a.txt', '-', {}); + + expect(chunks.join('')).toBe('hello'); + }); + + it('reads stdin into a remote file', async () => { + const { client, calls } = makeClient({ entries: {} }); + state.client = client; + const { Readable } = await import('node:stream'); + const stdin = Object.getOwnPropertyDescriptor(process, 'stdin'); + Object.defineProperty(process, 'stdin', { + value: Readable.from([Buffer.from('piped')]), + configurable: true, + }); + + try { + await fsCp('-', 'puter:/from-stdin.txt', {}); + } finally { + Object.defineProperty(process, 'stdin', stdin); + } + + expect(firstOf(calls, 'write').args[0]).toBe('~/from-stdin.txt'); + expect(firstOf(calls, 'write').args[1]).toBe(5); + }); +}); + +describe('fs ls', () => { + it('emits operands that feed back into another command when piped', async () => { + const { client } = makeClient({ + entries: { '~/logs': dir('~/logs') }, + listings: { '~/logs': [entry('~/logs/a.log'), dir('~/logs/old')] }, + }); + state.client = client; + + await fsLs('puter:/logs', {}); + + expect(out).toEqual(['puter:/logs/a.log', 'puter:/logs/old']); + }); + + it('falls back to the full listing when the SDK cannot stream', async () => { + const { client } = makeClient({ + listings: { '~/logs': [entry('~/logs/a.log')] }, + }); + // An SDK without a streaming readdir answers the same call with the + // whole listing instead of an async iterator. + const inner = client.fs.readdir; + client.fs.readdir = (arg) => { + const options = typeof arg === 'string' ? { path: arg } : arg; + return Promise.resolve(inner({ ...options, stream: false })); + }; + state.client = client; + + await fsLs('puter:/logs', {}); + + expect(out).toEqual(['puter:/logs/a.log']); + }); + + it('lines up type, size and time with -l', async () => { + const { client } = makeClient({ + listings: { + '~/logs': [ + entry('~/logs/a.log', { size: 2048 }), + dir('~/logs/old'), + ], + }, + }); + state.client = client; + + await fsLs('puter:/logs', { long: true }); + + // The size column is padded to the widest value, so both rows line up. + expect(out).toHaveLength(2); + expect(out[0]).toMatch(/^- 2\.0 KB \d{4}-\d{2}-\d{2} \d{2}:\d{2} a\.log$/); + expect(out[1]).toMatch(/^d {6}- \d{4}-\d{2}-\d{2} \d{2}:\d{2} old\/$/); + }); + + it('gives the whole entry with --json', async () => { + const { client } = makeClient({ + listings: { '~/logs': [entry('~/logs/a.log')] }, + }); + state.client = client; + + await fsLs('puter:/logs', { json: true }); + + expect(JSON.parse(out.join('\n'))).toEqual([entry('~/logs/a.log')]); + }); +}); + +describe('fs stat and cat', () => { + it('prints one field per line, and everything with --json', async () => { + const { client } = makeClient({ entries: { '~/a.txt': entry('~/a.txt') } }); + state.client = client; + + await fsStat('puter:/a.txt', {}); + expect(out[0]).toBe('path: /u/a.txt'); + expect(out[1]).toBe('type: file'); + expect(out[2]).toBe('size: 1024 (1.0 KB)'); + + out.length = 0; + await fsStat('puter:/a.txt', { json: true }); + expect(JSON.parse(out.join('\n')).uid).toBe('uid-a.txt'); + }); + + it('refuses to cat a directory', async () => { + const { client } = makeClient({ entries: { '~/dist': dir('~/dist') } }); + state.client = client; + + await expect(fsCat('puter:/dist', {})).rejects.toThrow(/is a directory/); + }); +}); + +describe('fs mv', () => { + it('hands the destination to move(), which works out dir vs rename', async () => { + const { client, calls } = makeClient({ entries: { '~/a.txt': entry('~/a.txt') } }); + state.client = client; + + await fsMv('puter:/a.txt', 'puter:/Desktop/b.txt', {}); + + expect(firstOf(calls, 'move').args).toEqual([ + '~/a.txt', + '~/Desktop/b.txt', + { overwrite: true, dedupeName: false }, + ]); + expect(callsOf(calls, 'rename')).toHaveLength(0); + }); + + it('refuses to move the root', async () => { + const { client, calls } = makeClient({ entries: { '~': dir('~') } }); + state.client = client; + + await expect(fsMv('puter:/', 'puter:/x', {})).rejects.toThrow(/the root itself/); + expect(callsOf(calls, 'move')).toHaveLength(0); + }); +}); diff --git a/src/cli/src/commands/kv.js b/src/cli/src/commands/kv.js index dd8532955..f240af706 100644 --- a/src/cli/src/commands/kv.js +++ b/src/cli/src/commands/kv.js @@ -11,10 +11,10 @@ import path from 'node:path'; import repl from 'node:repl'; -import { appsApi } from '../lib/apps.js'; +import { resolveTarget } from '../lib/appTarget.js'; import { ensureClient } from '../lib/auth.js'; import { configPath } from '../lib/config.js'; -import { isInteractive, WORKER_DOMAIN } from '../lib/env.js'; +import { isInteractive } from '../lib/env.js'; import { CLIError, messageOf } from '../lib/errors.js'; import { bindApp, KV_METHODS } from '../lib/kvbind.js'; import * as ui from '../lib/ui.js'; @@ -24,101 +24,6 @@ import * as ui from '../lib/ui.js'; // instead and say "100+" when it fills. const PROBE_LIMIT = 100; -const APPS_HINT = "Run 'puter app list' to see your apps."; -const WORKERS_HINT = "Run 'puter worker list' to see your workers."; -const TARGET_HINT = - "Run 'puter app list' or 'puter worker list' to see what you can connect to."; - -// A worker URL identifies a worker as well as its name does — copy one out of -// `puter worker list` or the browser and it connects. Returns the worker name, -// or null when the argument isn't a worker URL. -function workerNameFromUrl(arg) { - const host = String(arg) - .trim() - .replace(/^[a-z]+:\/\//i, '') // scheme - .replace(/[/?#].*$/, '') // path, query, fragment - .replace(/:\d+$/, '') // port - .toLowerCase(); - - const suffix = `.${WORKER_DOMAIN.toLowerCase()}`; - if (!host.endsWith(suffix)) return null; - const name = host.slice(0, -suffix.length); - // Only the flat `.puter.work` form — a deeper host isn't one of ours. - return name && !name.includes('.') ? name : null; -} - -// "No such app" is the cue to try a worker by the same name; anything else -// (offline, expired token) is a real failure and should be reported as one. -const isNotFound = (err) => - err?.code === 'not_found' || /not found/i.test(messageOf(err)); - -// A worker deployed from a user token gets its own `sandbox-` app, and -// with it a key-value store of its own — so a worker name resolves to the uid -// of that app. Returns null when the account has no worker by that name. -async function resolveWorker(puter, nameArg) { - let worker; - try { - worker = await puter.workers.get(nameArg); - } catch (err) { - throw new CLIError( - `Could not fetch worker '${nameArg}': ${messageOf(err)}`, - { hint: WORKERS_HINT }, - ); - } - if (!worker) return null; - - // Workers deployed by an app (or with `sandbox: false`) have no app identity - // of their own — they read and write the store of whoever deployed them. - if (!worker.app_uid) { - throw new CLIError( - `Worker '${worker.name ?? nameArg}' has no store of its own: it is not sandboxed.`, - { hint: 'Connect to the app that owns it instead.' }, - ); - } - return { name: worker.name ?? nameArg, uid: worker.app_uid, kind: 'worker' }; -} - -// Accept an app name, a worker name, a worker URL, or the uid itself, so a uid -// pasted from `puter app list` (or the MCP tools) doesn't need a lookup. An app -// name wins a tie with a worker of the same name — the worker's own URL is the -// way to ask for the worker instead. -async function resolveTarget(puter, identifier) { - // A URL says which namespace it belongs to, so it never falls back to an app. - const fromUrl = workerNameFromUrl(identifier); - if (fromUrl) { - const worker = await resolveWorker(puter, fromUrl); - if (!worker) { - throw new CLIError(`Worker '${fromUrl}' not found.`, { - hint: WORKERS_HINT, - }); - } - return worker; - } - - if (/^app-/.test(identifier)) { - return { name: identifier, uid: identifier, byUid: true }; - } - - let app; - try { - app = await appsApi(puter).get(identifier); - } catch (err) { - if (!isNotFound(err)) { - throw new CLIError(`Could not fetch '${identifier}': ${messageOf(err)}`, { - hint: APPS_HINT, - }); - } - } - if (app?.uid) return { name: app.name ?? identifier, uid: app.uid }; - - const worker = await resolveWorker(puter, identifier); - if (worker) return worker; - - throw new CLIError(`No app or worker named '${identifier}'.`, { - hint: TARGET_HINT, - }); -} - // One round-trip that both proves the store is reachable and gives us // something to put in the banner. async function probeKeys(kv, label) { diff --git a/src/cli/src/commands/site.js b/src/cli/src/commands/site.js index 8d365c4a7..c4ccb0b7c 100644 --- a/src/cli/src/commands/site.js +++ b/src/cli/src/commands/site.js @@ -6,7 +6,8 @@ import { isInteractive, SITE_DOMAIN } from '../lib/env.js'; import { ensureClient } from '../lib/auth.js'; import { randName } from '../lib/client.js'; import { walk } from '../lib/fswalk.js'; -import { CLIError } from '../lib/errors.js'; +import { CLIError, messageOf } from '../lib/errors.js'; +import { uploadFiles } from '../lib/transfer.js'; import * as ui from '../lib/ui.js'; // --- subdomain helpers (spec §5.3): liberal in, strict out ---------------- @@ -44,16 +45,6 @@ function isIgnored(relPath) { return IGNORED_NAMES.has(relPath.split('/').pop()); } -// Build a File the SDK's upload() accepts, preserving the file's relative path -// so nested directories are recreated. The SDK overwrites .filepath/.fullPath -// with the basename but reads .finalPath first, so that's where the rel path -// has to go. -function toUploadFile(buf, relPath) { - const file = new File([buf], relPath.split('/').pop()); - file.finalPath = relPath; - return file; -} - // --- deploy (spec §5.5) --------------------------------------------------- export async function siteDeploy(dirArg, subArg, opts) { @@ -150,21 +141,36 @@ export async function siteDeploy(dirArg, subArg, opts) { ui.debug('mkdir returned:', JSON.stringify(folder)); ui.debug('target path:', targetPath); - // 4. Upload the whole tree in one batch. Each File carries its relative path - // (via finalPath) so nested folders are recreated under targetPath; - // createMissingParents builds those intermediate folders server-side. - const items = files.map((f) => toUploadFile(fs.readFileSync(f.full), f.rel)); - const sp = ui.spinner(`Uploading ${items.length} file(s)...`); + // 4. Upload the tree one directory at a time, so subfolders are recreated + // under targetPath. On Node the batch endpoint cannot build a directory + // tree, so the nesting has to come from each upload's own dirPath. + const sp = ui.spinner(`Uploading ${files.length} file(s)...`); + let uploaded; + let failures; try { - await puter.fs.upload(items, targetPath, { - overwrite: true, - createMissingParents: true, - }); - sp.stop(`Uploaded ${items.length} file(s).`); + ({ uploaded, failures } = await uploadFiles(puter, { + files, + destination: targetPath, + onProgress: (done, total) => sp.message(`Uploaded ${done}/${total} file(s)...`), + onRetry: (err, attempt) => ui.debug(`retry ${attempt}:`, messageOf(err)), + })); + sp.stop(`Uploaded ${uploaded} of ${files.length} file(s).`); } catch (err) { sp.stop('Upload failed.'); ui.debug('upload error object:', JSON.stringify(err)); - throw new CLIError(`Upload failed: ${err.message ?? JSON.stringify(err)}`); + throw new CLIError(`Upload failed: ${messageOf(err)}`); + } + + // A site missing files is a broken site, so a partial upload fails here — + // before the subdomain is pointed at it. + if (failures.length > 0) { + for (const failure of failures.slice(0, 10)) { + ui.warn(`${failure.rel}: ${failure.message}`); + } + if (failures.length > 10) ui.info(`... and ${failures.length - 10} more.`); + throw new CLIError( + `${failures.length} of ${files.length} file(s) failed to upload; nothing was deployed.`, + ); } // 5. Point the subdomain at the new folder. diff --git a/src/cli/src/lib/appTarget.js b/src/cli/src/lib/appTarget.js new file mode 100644 index 000000000..5dc76f8f0 --- /dev/null +++ b/src/cli/src/lib/appTarget.js @@ -0,0 +1,103 @@ +// Resolving "which app" from something a person can type: an app name, a +// worker name, a worker URL, or a uid. Shared by `puter kv connect` and +// `puter fs --app`, so both accept the same identifiers. + +import { appsApi } from './apps.js'; +import { WORKER_DOMAIN } from './env.js'; +import { CLIError, messageOf } from './errors.js'; + +const APPS_HINT = "Run 'puter app list' to see your apps."; +const WORKERS_HINT = "Run 'puter worker list' to see your workers."; +const TARGET_HINT = + "Run 'puter app list' or 'puter worker list' to see what you can connect to."; + +// A worker URL identifies a worker as well as its name does — copy one out of +// `puter worker list` or the browser and it resolves. Returns the worker name, +// or null when the argument isn't a worker URL. +export function workerNameFromUrl(arg) { + const host = String(arg) + .trim() + .replace(/^[a-z]+:\/\//i, '') // scheme + .replace(/[/?#].*$/, '') // path, query, fragment + .replace(/:\d+$/, '') // port + .toLowerCase(); + + const suffix = `.${WORKER_DOMAIN.toLowerCase()}`; + if (!host.endsWith(suffix)) return null; + const name = host.slice(0, -suffix.length); + // Only the flat `.puter.work` form — a deeper host isn't one of ours. + return name && !name.includes('.') ? name : null; +} + +// "No such app" is the cue to try a worker by the same name; anything else +// (offline, expired token) is a real failure and should be reported as one. +const isNotFound = (err) => + err?.code === 'not_found' || /not found/i.test(messageOf(err)); + +// A worker deployed from a user token gets its own `sandbox-` app, and +// with it a key-value store and AppData directory of its own — so a worker +// name resolves to the uid of that app. Returns null when the account has no +// worker by that name. +async function resolveWorker(puter, nameArg) { + let worker; + try { + worker = await puter.workers.get(nameArg); + } catch (err) { + throw new CLIError( + `Could not fetch worker '${nameArg}': ${messageOf(err)}`, + { hint: WORKERS_HINT }, + ); + } + if (!worker) return null; + + // Workers deployed by an app (or with `sandbox: false`) have no app identity + // of their own — they read and write the storage of whoever deployed them. + if (!worker.app_uid) { + throw new CLIError( + `Worker '${worker.name ?? nameArg}' has no storage of its own: it is not sandboxed.`, + { hint: 'Use the app that owns it instead.' }, + ); + } + return { name: worker.name ?? nameArg, uid: worker.app_uid, kind: 'worker' }; +} + +// Accept an app name, a worker name, a worker URL, or the uid itself, so a uid +// pasted from `puter app list` (or the MCP tools) doesn't need a lookup. An app +// name wins a tie with a worker of the same name — the worker's own URL is the +// way to ask for the worker instead. +export async function resolveTarget(puter, identifier) { + // A URL says which namespace it belongs to, so it never falls back to an app. + const fromUrl = workerNameFromUrl(identifier); + if (fromUrl) { + const worker = await resolveWorker(puter, fromUrl); + if (!worker) { + throw new CLIError(`Worker '${fromUrl}' not found.`, { + hint: WORKERS_HINT, + }); + } + return worker; + } + + if (/^app-/.test(identifier)) { + return { name: identifier, uid: identifier, byUid: true }; + } + + let app; + try { + app = await appsApi(puter).get(identifier); + } catch (err) { + if (!isNotFound(err)) { + throw new CLIError(`Could not fetch '${identifier}': ${messageOf(err)}`, { + hint: APPS_HINT, + }); + } + } + if (app?.uid) return { name: app.name ?? identifier, uid: app.uid }; + + const worker = await resolveWorker(puter, identifier); + if (worker) return worker; + + throw new CLIError(`No app or worker named '${identifier}'.`, { + hint: TARGET_HINT, + }); +} diff --git a/src/cli/src/lib/remotePath.js b/src/cli/src/lib/remotePath.js new file mode 100644 index 000000000..02c9641b7 --- /dev/null +++ b/src/cli/src/lib/remotePath.js @@ -0,0 +1,178 @@ +// Operand parsing for `puter fs`. +// +// Which side of the wire a path is on is decided by the operand itself: +// `puter:/...` is remote, `-` is stdin/stdout, anything else is local. Remote +// paths are absolute from the account's home directory — there is no remote +// working directory for a relative path to be relative to — and they are +// clamped to their root, so `..` is refused rather than quietly rebased. With +// `--app` that root is the app's storage directory, and a half-enforced +// sandbox would be worse than none. + +import { CLIError } from './errors.js'; + +const PREFIX = 'puter:'; + +// The account's home directory. The SDK expands `~` server-side; `/` on Puter +// is the system root (a listing of usernames), which is not what someone +// typing `puter:/` means. +export const HOME_ROOT = '~'; + +// An app's storage directory, keyed by uid — the same `~/AppData/` that +// puter.js resolves an app's relative paths against. +export function appRoot(uid) { + return `${HOME_ROOT}/AppData/${uid}`; +} + +export function isRemote(operand) { + return typeof operand === 'string' && operand.startsWith(PREFIX); +} + +export function isStdio(operand) { + return operand === '-'; +} + +export function kindOf(operand) { + if (isStdio(operand)) return 'stdio'; + if (isRemote(operand)) return 'remote'; + return 'local'; +} + +// The operand form of a resolved remote path, for output that gets piped back +// into another command. +export function toOperand(remotePath, root = HOME_ROOT) { + if (!remotePath.startsWith(root)) return remotePath; + const rest = remotePath.slice(root.length); + return `${PREFIX}${rest.startsWith('/') ? rest : `/${rest}`}`; +} + +/** + * The checks that don't depend on which root a path resolves against, so a + * malformed operand is refused before we bother authenticating. Returns the + * path's components. + * + * @param {string} operand + * @returns {string[]} + */ +export function assertRemoteOperand(operand) { + if (!isRemote(operand)) { + throw new CLIError(`'${operand}' is not a remote path.`, { + hint: `Remote paths start with '${PREFIX}/' — e.g. ${PREFIX}/Desktop.`, + }); + } + + const raw = operand.slice(PREFIX.length); + if (!raw.startsWith('/')) { + throw new CLIError(`Remote paths must be absolute: '${operand}'.`, { + hint: `Write '${PREFIX}/${raw}' — there is no remote working directory for a relative path to resolve against.`, + }); + } + + const parts = []; + for (const part of raw.split('/')) { + if (part === '' || part === '.') continue; + if (part === '..') { + throw new CLIError( + `Remote paths cannot contain '..': '${operand}'.`, + { hint: `Write the path you mean out in full from ${PREFIX}/.` }, + ); + } + parts.push(part); + } + + return parts; +} + +/** + * Resolve a `puter:`-prefixed operand to an absolute remote path under `root`. + * + * @param {string} operand + * @param {string} [root] + * @returns {string} + */ +export function resolveRemote(operand, root = HOME_ROOT) { + const parts = assertRemoteOperand(operand); + return parts.length > 0 ? `${root}/${parts.join('/')}` : root; +} + +// Whether a resolved path *is* the root — the guard destructive commands need, +// since no legitimate one-liner empties a whole drive or app store. +export function isRoot(remotePath, root = HOME_ROOT) { + return remotePath === root; +} + +// Remote paths are POSIX regardless of the local platform, so the path +// helpers here are deliberately not node:path (which is `\`-flavoured on +// Windows). +export function remoteBasename(remotePath) { + const parts = remotePath.split('/'); + return parts[parts.length - 1]; +} + +export function remoteDirname(remotePath) { + const parts = remotePath.split('/'); + parts.pop(); + return parts.join('/') || HOME_ROOT; +} + +export function remoteJoin(remotePath, ...rest) { + return [remotePath.replace(/\/+$/, ''), ...rest].join('/'); +} + +const NEEDS_ONE_REMOTE = + `One side must be a remote '${PREFIX}/' path.`; + +/** + * Which implementation a `cp` operand pair calls for. Ambiguous pairs are + * refused loudly rather than guessed at. + * + * @param {string} source + * @param {string} destination + * @returns {'upload' | 'download' | 'remoteCopy' | 'writeStdin' | 'readStdout'} + */ +export function copyDirection(source, destination) { + const from = kindOf(source); + const to = kindOf(destination); + + if (from === 'local' && to === 'remote') return 'upload'; + if (from === 'remote' && to === 'local') return 'download'; + if (from === 'remote' && to === 'remote') return 'remoteCopy'; + if (from === 'stdio' && to === 'remote') return 'writeStdin'; + if (from === 'remote' && to === 'stdio') return 'readStdout'; + + if (from === 'local' && to === 'local') { + throw new CLIError( + `Both paths are local — copying '${source}' to '${destination}' is a job for cp.`, + { hint: NEEDS_ONE_REMOTE }, + ); + } + throw new CLIError( + `Cannot copy ${from === 'stdio' ? 'stdin' : from} to ${to === 'stdio' ? 'stdout' : to}.`, + { hint: NEEDS_ONE_REMOTE }, + ); +} + +/** + * `mv` is remote-to-remote only in v0: deleting local files after a failed + * upload is not a first impression worth making. + * + * @param {string} source + * @param {string} destination + */ +export function assertMovable(source, destination) { + const from = kindOf(source); + const to = kindOf(destination); + if (from === 'remote' && to === 'remote') return; + + if (from === 'stdio' || to === 'stdio') { + throw new CLIError('mv does not read stdin or write stdout.'); + } + if (from === 'local' && to === 'local') { + throw new CLIError( + `Both paths are local — moving '${source}' to '${destination}' is a job for mv.`, + { hint: NEEDS_ONE_REMOTE }, + ); + } + throw new CLIError('mv cannot cross between local and remote.', { + hint: 'Copy it with `puter fs cp`, check the result, then remove the original.', + }); +} diff --git a/src/cli/src/lib/remotePath.test.js b/src/cli/src/lib/remotePath.test.js new file mode 100644 index 000000000..40f9e7cae --- /dev/null +++ b/src/cli/src/lib/remotePath.test.js @@ -0,0 +1,116 @@ +import { describe, expect, it } from 'vitest'; + +import { + appRoot, + assertMovable, + copyDirection, + HOME_ROOT, + isRoot, + kindOf, + remoteBasename, + remoteDirname, + remoteJoin, + resolveRemote, + toOperand, +} from './remotePath.js'; + +describe('kindOf', () => { + it('reads the side of the wire off the operand', () => { + expect(kindOf('puter:/Desktop')).toBe('remote'); + expect(kindOf('-')).toBe('stdio'); + expect(kindOf('./dist')).toBe('local'); + expect(kindOf('dist')).toBe('local'); + // A local file that happens to be named like the prefix is still local. + expect(kindOf('/tmp/puter')).toBe('local'); + }); +}); + +describe('resolveRemote', () => { + it('resolves against the home directory, not the system root', () => { + expect(resolveRemote('puter:/Desktop/notes.txt')).toBe('~/Desktop/notes.txt'); + expect(resolveRemote('puter:/')).toBe(HOME_ROOT); + }); + + it('collapses empty and current-directory components', () => { + expect(resolveRemote('puter://a//b/./c')).toBe('~/a/b/c'); + }); + + it('rebases onto an app store without changing the operand', () => { + const root = appRoot('app-1f2e'); + expect(resolveRemote('puter:/dist', root)).toBe('~/AppData/app-1f2e/dist'); + expect(resolveRemote('puter:/', root)).toBe('~/AppData/app-1f2e'); + }); + + it('refuses to climb out of the root instead of rebasing quietly', () => { + expect(() => resolveRemote('puter:/../../Documents', appRoot('app-1'))).toThrow(/cannot contain/); + expect(() => resolveRemote('puter:/dist/../..')).toThrow(/cannot contain/); + }); + + it('refuses relative remote paths, which have nothing to be relative to', () => { + expect(() => resolveRemote('puter:dist')).toThrow(/must be absolute/); + }); + + it('refuses an operand that is not remote at all', () => { + expect(() => resolveRemote('./dist')).toThrow(/not a remote path/); + }); +}); + +describe('isRoot', () => { + it('recognizes the root of whichever tree is in play', () => { + expect(isRoot(resolveRemote('puter:/'))).toBe(true); + expect(isRoot(resolveRemote('puter:/x'))).toBe(false); + + const root = appRoot('app-1'); + expect(isRoot(resolveRemote('puter:/', root), root)).toBe(true); + expect(isRoot(resolveRemote('puter:/dist', root), root)).toBe(false); + }); +}); + +describe('toOperand', () => { + it('round-trips a resolved path back into something a command accepts', () => { + expect(toOperand('~/logs/a.txt')).toBe('puter:/logs/a.txt'); + expect(toOperand(HOME_ROOT)).toBe('puter:/'); + + const root = appRoot('app-1'); + expect(toOperand(`${root}/dist/app.js`, root)).toBe('puter:/dist/app.js'); + }); +}); + +describe('remote path helpers', () => { + it('stays POSIX regardless of the local platform', () => { + expect(remoteBasename('~/a/b/c.txt')).toBe('c.txt'); + expect(remoteDirname('~/a/b/c.txt')).toBe('~/a/b'); + expect(remoteDirname('~/a')).toBe(HOME_ROOT); + expect(remoteJoin('~/a/', 'b', 'c')).toBe('~/a/b/c'); + }); +}); + +describe('copyDirection', () => { + it('picks the implementation from the operand pair', () => { + expect(copyDirection('./dist', 'puter:/dist')).toBe('upload'); + expect(copyDirection('puter:/dist', './dist')).toBe('download'); + expect(copyDirection('puter:/a', 'puter:/b')).toBe('remoteCopy'); + expect(copyDirection('-', 'puter:/a')).toBe('writeStdin'); + expect(copyDirection('puter:/a', '-')).toBe('readStdout'); + }); + + it('refuses the ambiguous pairs loudly', () => { + expect(() => copyDirection('./a', './b')).toThrow(/job for cp/); + expect(() => copyDirection('-', '-')).toThrow(/Cannot copy stdin to stdout/); + expect(() => copyDirection('-', './a')).toThrow(/Cannot copy/); + expect(() => copyDirection('./a', '-')).toThrow(/Cannot copy/); + }); +}); + +describe('assertMovable', () => { + it('allows remote to remote', () => { + expect(() => assertMovable('puter:/a', 'puter:/b')).not.toThrow(); + }); + + it('refuses to cross the boundary rather than half-implement it', () => { + expect(() => assertMovable('./a', 'puter:/b')).toThrow(/cannot cross/); + expect(() => assertMovable('puter:/a', './b')).toThrow(/cannot cross/); + expect(() => assertMovable('./a', './b')).toThrow(/job for mv/); + expect(() => assertMovable('-', 'puter:/b')).toThrow(/does not read stdin/); + }); +}); diff --git a/src/cli/src/lib/transfer.js b/src/cli/src/lib/transfer.js new file mode 100644 index 000000000..bdedb8ab2 --- /dev/null +++ b/src/cli/src/lib/transfer.js @@ -0,0 +1,253 @@ +// Moving bytes in bulk: bounded concurrency, retry with backoff, and the +// batching that `puter.fs.upload()` wants. +// +// Everything here goes through the SDK rather than talking to the API +// directly — `upload()` is the fast path (one signed batch write per call +// instead of a request per file), and it reports which of its items failed, +// so a retry re-sends only those. + +import fs from 'node:fs'; + +import { messageOf } from './errors.js'; + +export const DEFAULT_CONCURRENCY = 8; + +// A batch is capped by both count and size because its files are held in +// memory at once, and `--concurrency` batches are in flight at a time. +const MAX_BATCH_FILES = 100; +const MAX_BATCH_BYTES = 8 * 1024 * 1024; + +const ATTEMPTS = 3; +const BASE_BACKOFF_MS = 500; + +// puter.fs.read() resolves a whole Blob — the SDK has no streaming read — so +// anything past this is pulled in ranged chunks to keep memory flat. +const CHUNKED_READ_THRESHOLD = 32 * 1024 * 1024; +const READ_CHUNK_BYTES = 8 * 1024 * 1024; + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +// Worth another go: the request never reached a verdict (network) or the +// server said it couldn't answer right now (5xx, 429). A 4xx won't change +// its mind, and retrying it only delays the message. +export function isRetryable(err) { + const status = Number(err?.status ?? err?.statusCode); + if (Number.isFinite(status) && status > 0) { + return status >= 500 || status === 429; + } + const code = err?.code ?? err?.errno; + if (typeof code === 'string' && /^(ECONN|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|EPIPE|ENETUNREACH|ESOCKET)/.test(code)) { + return true; + } + return /network|socket hang up|timed? ?out|fetch failed/i.test(messageOf(err)); +} + +export async function withRetry(fn, { attempts = ATTEMPTS, onRetry } = {}) { + for (let attempt = 1; ; attempt++) { + try { + return await fn(); + } catch (err) { + if (attempt >= attempts || !isRetryable(err)) throw err; + const delay = BASE_BACKOFF_MS * 2 ** (attempt - 1); + onRetry?.(err, attempt, delay); + await sleep(delay); + } + } +} + +/** + * Run `worker` over `items` with at most `limit` in flight, preserving input + * order in the returned results. Rejections are the worker's to handle — this + * resolves with whatever each call returned. + * + * @template T, R + * @param {T[]} items + * @param {number} limit + * @param {(item: T, index: number) => Promise} worker + * @returns {Promise} + */ +export async function pool(items, limit, worker) { + const results = new Array(items.length); + let next = 0; + const runners = new Array(Math.min(Math.max(1, limit), items.length || 1)) + .fill(null) + .map(async () => { + while (next < items.length) { + const index = next++; + results[index] = await worker(items[index], index); + } + }); + await Promise.all(runners); + return results; +} + +function sizeOf(file) { + if (Number.isFinite(file.size)) return file.size; + try { + return fs.statSync(file.full).size; + } catch { + return 0; + } +} + +const relDirOf = (rel) => { + const slash = rel.lastIndexOf('/'); + return slash === -1 ? '' : rel.slice(0, slash); +}; + +const relNameOf = (rel) => rel.slice(rel.lastIndexOf('/') + 1); + +// Batches are grouped by directory, and each one uploads into its own +// `dirPath`, because that is the only part of the destination the SDK +// honors: the per-file `finalPath` it computes a nested path from is dropped +// on the signed batch-write path, which collapses a tree onto its basenames +// (and silently overwrites same-named files from different directories). +function batchFiles(files) { + const byDir = new Map(); + for (const file of files) { + const dir = relDirOf(file.rel); + if (!byDir.has(dir)) byDir.set(dir, []); + byDir.get(dir).push(file); + } + + const batches = []; + for (const [dir, group] of byDir) { + let current = []; + let bytes = 0; + for (const file of group) { + const size = sizeOf(file); + if ( + current.length > 0 && + (current.length >= MAX_BATCH_FILES || bytes + size > MAX_BATCH_BYTES) + ) { + batches.push({ dir, files: current }); + current = []; + bytes = 0; + } + current.push(file); + bytes += size; + } + if (current.length > 0) batches.push({ dir, files: current }); + } + return batches; +} + +// upload() rejects with one of two shapes depending on which path ran: the +// signed batch write reports `failedPaths`/`failedItems`, the legacy /batch +// path a `batch_upload_*` code with `failedItems`. Both name the operations +// that failed, so pull out the subset still owed and leave the rest counted +// as written. Returns null when the failure names nothing. +function attributeFailures(pending, err) { + const named = [ + ...(Array.isArray(err?.failedPaths) ? err.failedPaths : []), + ...(Array.isArray(err?.failedItems) ? err.failedItems : []).map( + (item) => item?.path ?? item?.name, + ), + ].filter((value) => typeof value === 'string' && value.length > 0); + + if (named.length === 0) return null; + + // The server reports absolute paths; ours are relative to the destination. + const failed = pending.filter((file) => + named.some((p) => p === file.rel || p.endsWith(`/${file.rel}`)), + ); + return failed.length > 0 ? failed : null; +} + +async function sendBatch(puter, dir, files, destination, options) { + const items = files.map( + (file) => new File([fs.readFileSync(file.full)], relNameOf(file.rel)), + ); + await puter.fs.upload(items, dir ? `${destination}/${dir}` : destination, { + // `-n` filtered out what already exists, so a conflict here means the + // destination changed under us: fail it rather than clobber. + overwrite: !options.noClobber, + dedupeName: false, + createMissingParents: true, + }); +} + +/** + * Upload local files into a remote directory, preserving their relative paths. + * A file that keeps failing is collected rather than aborting the run — the + * caller reports the summary and exits non-zero. + * + * @returns {Promise<{ uploaded: number, failures: {rel: string, message: string}[] }>} + */ +export async function uploadFiles( + puter, + { files, destination, concurrency = DEFAULT_CONCURRENCY, noClobber = false, onProgress, onRetry }, +) { + const failures = []; + let uploaded = 0; + + await pool(batchFiles(files), concurrency, async (batch) => { + let pending = batch.files; + for (let attempt = 1; attempt <= ATTEMPTS; attempt++) { + try { + await sendBatch(puter, batch.dir, pending, destination, { noClobber }); + uploaded += pending.length; + onProgress?.(uploaded + failures.length, files.length); + return; + } catch (err) { + // Items the server did write don't need re-sending; when the failure + // names nothing, the whole batch is still owed. + const owed = attributeFailures(pending, err) ?? pending; + uploaded += pending.length - owed.length; + + if (attempt === ATTEMPTS || !isRetryable(err)) { + const message = messageOf(err); + for (const file of owed) failures.push({ rel: file.rel, message }); + onProgress?.(uploaded + failures.length, files.length); + return; + } + onRetry?.(err, attempt); + pending = owed; + await sleep(BASE_BACKOFF_MS * 2 ** (attempt - 1)); + } + } + }); + + return { uploaded, failures }; +} + +/** + * Read a remote file as a sequence of buffers. Files past the chunk threshold + * are pulled with `offset`/`byte_count` (which the backend serves as a Range + * request) so memory stays flat; if a chunk comes back longer than asked for + * the range wasn't honored, and that response is the whole file. + * + * @param {*} puter + * @param {string} remotePath + * @param {number} [size] + * @returns {AsyncGenerator} + */ +export async function* readRemote(puter, remotePath, size) { + const toBuffer = async (blob) => Buffer.from(await blob.arrayBuffer()); + + if (!Number.isFinite(size) || size <= CHUNKED_READ_THRESHOLD) { + yield await toBuffer(await withRetry(() => puter.fs.read(remotePath))); + return; + } + + let offset = 0; + while (offset < size) { + const byteCount = Math.min(READ_CHUNK_BYTES, size - offset); + const chunk = await toBuffer( + await withRetry(() => + puter.fs.read(remotePath, { offset, byte_count: byteCount }), + ), + ); + yield chunk; + if (chunk.length === 0 || chunk.length > byteCount) return; + offset += chunk.length; + } +} + +// Write to a stream and wait for it to drain, so a large transfer doesn't +// queue the whole file in memory behind a slow consumer. +export function writeChunk(stream, buf) { + return new Promise((resolve, reject) => { + stream.write(buf, (err) => (err ? reject(err) : resolve())); + }); +} diff --git a/src/cli/src/lib/transfer.test.js b/src/cli/src/lib/transfer.test.js new file mode 100644 index 000000000..04616ab50 --- /dev/null +++ b/src/cli/src/lib/transfer.test.js @@ -0,0 +1,273 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { isRetryable, pool, readRemote, uploadFiles, withRetry } from './transfer.js'; + +const tempDirs = []; + +function makeLocalFiles(count, bytes = 8) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'puter-cli-transfer-')); + tempDirs.push(dir); + const files = []; + for (let i = 0; i < count; i++) { + const rel = `f${i}.txt`; + const full = path.join(dir, rel); + fs.writeFileSync(full, Buffer.alloc(bytes, 1)); + files.push({ full, rel }); + } + return files; +} + +afterEach(() => { + while (tempDirs.length > 0) { + fs.rmSync(tempDirs.pop(), { recursive: true, force: true }); + } +}); + +describe('isRetryable', () => { + it('retries what might answer differently next time', () => { + expect(isRetryable({ status: 500 })).toBe(true); + expect(isRetryable({ status: 503 })).toBe(true); + expect(isRetryable({ status: 429 })).toBe(true); + expect(isRetryable({ code: 'ECONNRESET' })).toBe(true); + expect(isRetryable({ message: 'fetch failed' })).toBe(true); + }); + + it('does not retry a verdict', () => { + expect(isRetryable({ status: 404 })).toBe(false); + expect(isRetryable({ status: 403 })).toBe(false); + expect(isRetryable({ status: 400, code: 'invalid_request' })).toBe(false); + }); +}); + +describe('withRetry', () => { + it('gives up immediately on a client error', async () => { + let calls = 0; + await expect( + withRetry(() => { + calls++; + return Promise.reject({ status: 400, message: 'nope' }); + }), + ).rejects.toMatchObject({ status: 400 }); + expect(calls).toBe(1); + }); + + it('retries a server error and returns the eventual success', async () => { + let calls = 0; + const result = await withRetry(() => { + calls++; + if (calls < 3) return Promise.reject({ status: 500 }); + return Promise.resolve('ok'); + }); + expect(result).toBe('ok'); + expect(calls).toBe(3); + }); +}); + +describe('pool', () => { + it('keeps at most `limit` in flight and preserves result order', async () => { + let inFlight = 0; + let peak = 0; + const items = Array.from({ length: 20 }, (_, i) => i); + + const results = await pool(items, 4, async (item) => { + peak = Math.max(peak, ++inFlight); + await new Promise((resolve) => setTimeout(resolve, 1)); + inFlight--; + return item * 2; + }); + + expect(peak).toBeLessThanOrEqual(4); + expect(results).toEqual(items.map((i) => i * 2)); + }); +}); + +describe('uploadFiles', () => { + function client(upload) { + return { fs: { upload } }; + } + + it('batches by file count rather than sending one request per file', async () => { + const files = makeLocalFiles(101); + const batches = []; + const puter = client(async (items) => { + batches.push(items.length); + }); + + const result = await uploadFiles(puter, { + files, + destination: '~/dst', + concurrency: 1, + }); + + expect(batches).toEqual([100, 1]); + expect(result).toEqual({ uploaded: 101, failures: [] }); + }); + + it('carries nesting in the destination, not in a per-file path', async () => { + // The SDK drops the per-file relative path on its signed batch-write + // path, so each directory is uploaded into its own dirPath instead. + const files = makeLocalFiles(1); + files.push( + { full: files[0].full, rel: 'nested/c.txt' }, + { full: files[0].full, rel: 'nested/deep/d.txt' }, + { full: files[0].full, rel: 'other/c.txt' }, + ); + const sent = []; + const puter = client(async (items, destination) => { + sent.push({ destination, names: items.map((item) => item.name) }); + }); + + await uploadFiles(puter, { files, destination: '~/dst', concurrency: 1 }); + + expect(sent).toEqual([ + { destination: '~/dst', names: ['f0.txt'] }, + { destination: '~/dst/nested', names: ['c.txt'] }, + { destination: '~/dst/nested/deep', names: ['d.txt'] }, + { destination: '~/dst/other', names: ['c.txt'] }, + ]); + }); + + it('retries a batch that failed for a reason that might not repeat', async () => { + const files = makeLocalFiles(3); + let calls = 0; + const puter = client(async () => { + if (++calls === 1) throw { status: 503, message: 'unavailable' }; + }); + + const result = await uploadFiles(puter, { files, destination: '~/dst' }); + + expect(calls).toBe(2); + expect(result.uploaded).toBe(3); + expect(result.failures).toEqual([]); + }); + + it('re-sends only the items a partial failure names', async () => { + const files = makeLocalFiles(3); + const sent = []; + let calls = 0; + const puter = client(async (items) => { + sent.push(items.map((item) => item.name)); + if (++calls === 1) { + throw { + partial: true, + status: 503, + message: 'one item failed', + failedPaths: ['/u/dst/f1.txt'], + }; + } + }); + + const result = await uploadFiles(puter, { files, destination: '~/dst' }); + + expect(sent[1]).toEqual(['f1.txt']); + expect(result).toEqual({ uploaded: 3, failures: [] }); + }); + + it('collects what it cannot upload instead of aborting the run', async () => { + const files = makeLocalFiles(3); + let calls = 0; + const puter = client(async () => { + calls++; + throw { + code: 'batch_upload_partially_failed', + status: 400, + message: 'name too long', + failedItems: [{ path: '/u/dst/f1.txt' }], + }; + }); + + const result = await uploadFiles(puter, { files, destination: '~/dst' }); + + // A 400 is a verdict, so the batch is not re-sent... + expect(calls).toBe(1); + // ...and the two items the server did write still count as uploaded. + expect(result.uploaded).toBe(2); + expect(result.failures).toEqual([ + { rel: 'f1.txt', message: 'name too long' }, + ]); + }); + + it('blames the whole batch when the failure names nothing', async () => { + const files = makeLocalFiles(2); + const puter = client(async () => { + throw { code: 'batch_upload_failed', status: 413, message: 'too large' }; + }); + + const result = await uploadFiles(puter, { files, destination: '~/dst' }); + + expect(result.uploaded).toBe(0); + expect(result.failures.map((f) => f.rel).sort()).toEqual(['f0.txt', 'f1.txt']); + }); +}); + +describe('readRemote', () => { + const blobOf = (bytes) => new Blob([Buffer.alloc(bytes, 7)]); + + it('reads a small file in one request', async () => { + const reads = []; + const puter = { + fs: { + read: async (p, options) => { + reads.push(options); + return blobOf(64); + }, + }, + }; + + const chunks = []; + for await (const chunk of readRemote(puter, '~/a.bin', 64)) chunks.push(chunk); + + expect(reads).toEqual([undefined]); + expect(chunks).toHaveLength(1); + expect(chunks[0]).toHaveLength(64); + }); + + it('pulls a large file in ranged chunks so memory stays flat', async () => { + const size = 40 * 1024 * 1024; + const reads = []; + const puter = { + fs: { + read: async (p, options) => { + reads.push(options); + return blobOf(options.byte_count); + }, + }, + }; + + let total = 0; + for await (const chunk of readRemote(puter, '~/big.bin', size)) total += chunk.length; + + expect(total).toBe(size); + expect(reads).toHaveLength(5); + expect(reads.map((r) => r.offset)).toEqual([ + 0, + 8 * 1024 * 1024, + 16 * 1024 * 1024, + 24 * 1024 * 1024, + 32 * 1024 * 1024, + ]); + }); + + it('treats an over-long chunk as the whole file, not the first of many', async () => { + const size = 40 * 1024 * 1024; + let calls = 0; + const puter = { + fs: { + read: async () => { + calls++; + return blobOf(size); // a backend that ignored the range + }, + }, + }; + + const chunks = []; + for await (const chunk of readRemote(puter, '~/big.bin', size)) chunks.push(chunk); + + expect(calls).toBe(1); + expect(chunks).toHaveLength(1); + expect(chunks[0]).toHaveLength(size); + }); +}); diff --git a/src/cli/vitest.config.js b/src/cli/vitest.config.js new file mode 100644 index 000000000..4175afdbd --- /dev/null +++ b/src/cli/vitest.config.js @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + include: ['src/**/*.test.js'], + environment: 'node', + }, +}); diff --git a/src/docs/src/cli.md b/src/docs/src/cli.md index 487563da1..6bd422d6d 100644 --- a/src/docs/src/cli.md +++ b/src/docs/src/cli.md @@ -1,9 +1,9 @@ --- title: CLI -description: Manage your Puter resources directly from your terminal with the Puter CLI. Deploy static sites and serverless workers without leaving your shell. +description: Manage your Puter resources directly from your terminal with the Puter CLI. Deploy static sites and serverless workers, and work with your cloud files, without leaving your shell. --- -The [Puter CLI](https://www.npmjs.com/package/@heyputer/cli) lets you manage your Puter resources straight from the terminal: deploy static websites, ship serverless workers, inspect the apps registered to your account, and explore the key-value stores behind your apps and workers, all without leaving your shell. +The [Puter CLI](https://www.npmjs.com/package/@heyputer/cli) lets you manage your Puter resources straight from the terminal: deploy static websites, ship serverless workers, work with your cloud files, inspect the apps registered to your account, and explore the key-value stores behind your apps and workers, all without leaving your shell.
The Puter CLI is in beta (0.x). Behavior may change between releases.
@@ -81,6 +81,66 @@ puter app list # list your apps puter app get # show one app's details ``` +## Files + +Work with your [cloud storage](/FS/) from the terminal: list a directory, read a file, copy files and folders in either direction, move them, delete them, and inspect them. Remote paths carry a `puter:` prefix and are absolute from your home directory, `-` means stdin or stdout, and anything else is a local path — so the direction of a transfer comes from the paths themselves rather than from a flag. + +```sh +puter fs ls puter:/Desktop +puter fs cat puter:/notes.txt +puter fs cp -r ./dist puter:/Documents/backup +``` + +`cp` decides what to do from the pair of paths you give it: + +| From | To | What happens | +| --- | --- | --- | +| local | `puter:/…` | uploaded | +| `puter:/…` | local | downloaded | +| `puter:/…` | `puter:/…` | copied on the server, without passing through your machine | +| `-` | `puter:/…` | stdin is written to the file | +| `puter:/…` | `-` | the file's bytes are written to stdout | + +Two local paths — or two of anything else — is an error rather than a guess. `mv` works within your Puter storage only; to move something between your machine and Puter, copy it, check the copy, then delete the original. + +### Piping + +Status messages, progress and prompts go to stderr and data goes to stdout, so output can be piped without status text mixed into it: + +```sh +puter fs ls puter:/logs | xargs -n1 puter fs cat +``` + +In a terminal you get the readable view: bare names, aligned columns with `-l`, and a progress spinner during transfers. When output is piped or redirected, `ls` prints full `puter:` paths instead, one per line, so each line can be handed straight to another command. `--json` prints the complete entries, for both `ls` and `stat`. + +### App storage + +Every app has its own storage directory. `--app` resolves `puter:/` against that directory instead of your home directory, so you can read and edit the files an app works with. It accepts the same identifiers as `puter kv connect` — an app name, an app uid, a worker name, or a worker URL: + +```console +$ puter fs ls --app notes puter:/ +puter:/settings.json +``` + +`--app` is a flag and nothing else: there is no environment variable and no saved default, because it changes what an absolute path means. Every command that writes or deletes prints the path it resolved to, so you can see the difference it made: + +```console +$ puter fs rm -r --app notes puter:/cache +rm -r puter:/cache → ~/AppData/app-1f2e3d4c…/cache (412 entries) [notes (app-1f2e3d4c…)] +``` + +Paths stay inside that directory: `puter:/../../Documents` is an error, not a way out of it. + +### Deleting files + +`rm` deletes a single file. Deleting a directory needs `-r`, which prints the resolved path and how many entries it holds, then asks you to confirm in a terminal, or requires `--yes` when there is nobody to ask. `puter:/` on its own is always refused. Anything recursive accepts `--dry-run`, which lists what would be deleted without deleting it. + +
Deleted files do not go to Trash — puter fs rm removes them.
+ +### Transfers + +Copying a folder transfers 8 files at a time, which `--concurrency` adjusts (1–32), and retries a file whose failure looks temporary. If files still fail, the rest of the copy continues, the failures are listed at the end, and the command exits with a non-zero status — so a large upload does not start over because one file failed. `-n` skips files that already exist; without it, `cp` overwrites them. + ## Key-value store Open an interactive JavaScript shell against the [key-value store](/KV/) of one app or [worker](/Workers/), so you can read and edit its data directly instead of going through the app. Pass an app name, a worker name or its `*.puter.work` URL, or a uid: @@ -222,6 +282,88 @@ Show details for one app. | --- | --- | | `` | The app to inspect. | +### `puter fs ls` + +List a remote directory. + +| Argument / Option | Description | +| --- | --- | +| `` | The remote path to list (`puter:/…`). | +| `-l`, `--long` | Show type, size and modification time. | +| `--json` | Print the full entries as JSON. | +| `--app ` | Resolve `puter:/` against an app's storage instead of your home directory. | + +### `puter fs cat` + +Write a remote file's contents to stdout. + +| Argument / Option | Description | +| --- | --- | +| `` | The remote file to read (`puter:/…`). | +| `--app ` | Resolve `puter:/` against an app's storage instead of your home directory. | + +### `puter fs cp` + +Copy between your machine and Puter, or within your Puter storage. + +| Argument / Option | Description | +| --- | --- | +| `` | A local path, a remote path (`puter:/…`), or `-` to read stdin. | +| `` | A local path, a remote path (`puter:/…`), or `-` to write stdout. | +| `-r`, `--recursive` | Copy directories. | +| `-n`, `--no-clobber` | Skip files that already exist instead of overwriting them. | +| `--concurrency ` | How many files to transfer at once, from 1 to 32. Defaults to 8. | +| `--dry-run` | List what would be copied without copying it. | +| `--app ` | Resolve `puter:/` against an app's storage instead of your home directory. | + +One of the two paths must be remote. Copying between two local paths is an error. + +### `puter fs mv` + +Move or rename within your Puter storage. + +| Argument / Option | Description | +| --- | --- | +| `` | The remote path to move (`puter:/…`). | +| `` | The remote path to move it to (`puter:/…`). | +| `--app ` | Resolve `puter:/` against an app's storage instead of your home directory. | + +Both paths must be remote, and `puter:/` itself cannot be moved. + +### `puter fs rm` + +Delete a remote file or directory. + +| Argument / Option | Description | +| --- | --- | +| `` | The remote path to delete (`puter:/…`). | +| `-r`, `--recursive` | Delete a directory and everything in it. | +| `-y`, `--yes` | Skip the confirmation prompt. Required for `-r` when not running in a terminal. | +| `--dry-run` | List what would be deleted without deleting it. | +| `--app ` | Resolve `puter:/` against an app's storage instead of your home directory. | + +`puter:/` on its own is refused, with or without `--app`. + +### `puter fs mkdir` + +Create a remote directory. + +| Argument / Option | Description | +| --- | --- | +| `` | The remote directory to create (`puter:/…`). | +| `-p`, `--parents` | Create missing parent directories, and succeed if the directory already exists. | +| `--app ` | Resolve `puter:/` against an app's storage instead of your home directory. | + +### `puter fs stat` + +Show details for one remote file or directory. + +| Argument / Option | Description | +| --- | --- | +| `` | The remote path to inspect (`puter:/…`). | +| `--json` | Print the full entry as JSON. | +| `--app ` | Resolve `puter:/` against an app's storage instead of your home directory. | + ### `puter kv connect` Open an interactive shell against an app's or worker's key-value store.