diff --git a/src/backend/clients/dynamodb/DDBClient.ts b/src/backend/clients/dynamodb/DDBClient.ts index 7bee880df..ae6c41736 100644 --- a/src/backend/clients/dynamodb/DDBClient.ts +++ b/src/backend/clients/dynamodb/DDBClient.ts @@ -441,6 +441,7 @@ export class DDBClient extends PuterClient { index = '', consistentRead = false, options?: { + scanIndexForward?: boolean; beginsWith?: { key: string; value: string }; select?: 'COUNT'; filter?: { @@ -497,6 +498,9 @@ export class DDBClient extends PuterClient { ? { FilterExpression: options.filter.expression } : {}), ...(options?.select ? { Select: options.select } : {}), + ...(options?.scanIndexForward !== undefined + ? { ScanIndexForward: options.scanIndexForward } + : {}), ReturnConsumedCapacity: 'TOTAL', }); diff --git a/src/backend/drivers/kv/KVStoreDriver.test.ts b/src/backend/drivers/kv/KVStoreDriver.test.ts index 762651cec..374d055b1 100644 --- a/src/backend/drivers/kv/KVStoreDriver.test.ts +++ b/src/backend/drivers/kv/KVStoreDriver.test.ts @@ -52,6 +52,26 @@ describe('KVStoreDriver', () => { const inCtx = (fn: () => T | Promise, withActor: Actor = actor) => runWithContext({ actor: withActor }, fn); + it('mutates root array paths through the public driver', async () => { + await inCtx(async () => { + const key = 'rootArray'; + await target.set({ key, value: [{ score: 2, tags: [] }, 'keep'] }); + await target.update({ + key, + pathAndValueMap: { '[0].name': 'Puter' }, + }); + await target.incr({ key, pathAndAmountMap: { '[0].score': 3 } }); + await target.decr({ key, pathAndAmountMap: { '[0].score': 1 } }); + await target.add({ key, pathAndValueMap: { '[0].tags': ['new'] } }); + expect(await target.get({ key })).toEqual([ + { name: 'Puter', score: 4, tags: ['new'] }, + 'keep', + ]); + expect(await target.remove({ key, paths: ['[0]'] })).toEqual(['keep']); + expect(await target.get({ key })).toEqual(['keep']); + }); + }); + describe('get', () => { it('returns the value previously stored under the same key', async () => { const res = await inCtx(async () => { diff --git a/src/backend/drivers/kv/KVStoreDriver.ts b/src/backend/drivers/kv/KVStoreDriver.ts index 42db55906..904e40a16 100644 --- a/src/backend/drivers/kv/KVStoreDriver.ts +++ b/src/backend/drivers/kv/KVStoreDriver.ts @@ -431,6 +431,7 @@ export class KVStoreDriver extends PuterDriver { offset?: number; includeTotal?: boolean; fetchUntilFull?: boolean; + reverse?: boolean; optConfig?: { appUuid?: string }; }): Promise { const opts = await this.#opts('list', args); @@ -453,6 +454,7 @@ export class KVStoreDriver extends PuterDriver { offset: args.offset, includeTotal: args.includeTotal, fetchUntilFull: args.fetchUntilFull, + reverse: args.reverse, }, opts, ); diff --git a/src/backend/stores/systemKv/SystemKVStore.test.ts b/src/backend/stores/systemKv/SystemKVStore.test.ts index 3d350adff..1bb956c25 100644 --- a/src/backend/stores/systemKv/SystemKVStore.test.ts +++ b/src/backend/stores/systemKv/SystemKVStore.test.ts @@ -352,6 +352,96 @@ describe('SystemKVStore', () => { ); }); + it('lists backwards without changing the unpaginated shape', async () => { + expect( + (await target.list({ as: 'keys', reverse: true }, opts)).res, + ).toEqual(['veg:carrot', 'fruit:banana', 'fruit:apple']); + expect((await target.list({ as: 'keys' }, opts)).res).toEqual([ + 'fruit:apple', + 'fruit:banana', + 'veg:carrot', + ]); + }); + + it.each([false, true])( + 'preserves cursor direction reverse=%s', + async (reverse) => { + const first = ( + await target.list({ as: 'keys', limit: 1, reverse }, opts) + ).res as { items: string[]; cursor: string }; + expect(first.items).toEqual([ + reverse ? 'veg:carrot' : 'fruit:apple', + ]); + const second = ( + await target.list( + { as: 'keys', limit: 1, cursor: first.cursor }, + opts, + ) + ).res as { items: string[] }; + expect(second.items).toEqual(['fruit:banana']); + await expect( + target.list( + { limit: 1, cursor: first.cursor, reverse: !reverse }, + opts, + ), + ).rejects.toMatchObject({ statusCode: 400 }); + }, + ); + + it('applies reverse ordering to prefix filtering, offset, and totals', async () => { + const result = await target.list( + { + as: 'values', + pattern: 'fruit:*', + reverse: true, + offset: 1, + limit: 1, + includeTotal: true, + }, + opts, + ); + expect(result.res).toMatchObject({ items: ['red'], total: 2 }); + }); + + it('fills reverse pages past expired keys', async () => { + await target.set( + { + key: 'fruit:blueberry', + value: 'expired', + expireAt: Math.floor(Date.now() / 1000) - 10, + }, + opts, + ); + const result = await target.list( + { + as: 'keys', + pattern: 'fruit:*', + reverse: true, + limit: 2, + fetchUntilFull: true, + }, + opts, + ); + expect(result.res).toMatchObject({ + items: ['fruit:banana', 'fruit:apple'], + }); + }); + + it('rejects invalid reverse flags and cursor wrappers', async () => { + await expect( + target.list({ reverse: 'true' as unknown as boolean }, opts), + ).rejects.toMatchObject({ statusCode: 400 }); + for (const cursor of [ + { reverse: true, key: 'bad' }, + { reverse: true, key: {} }, + { reverse: false, key: {} }, + ]) { + await expect( + target.list({ cursor }, opts), + ).rejects.toMatchObject({ statusCode: 400 }); + } + }); + it('returns key/value entries by default', async () => { const result = await target.list({}, opts); expect(Array.isArray(result.res)).toBe(true); @@ -981,6 +1071,236 @@ describe('SystemKVStore', () => { }); }); + describe('document paths', () => { + const rootArrayOperations: Array<[ + string, + () => Promise<{ res: unknown }>, + unknown, + ]> = [ + [ + 'incr', + async () => + target.incr( + { key: 'root-incr', pathAndAmountMap: { '[0]': 1 } }, + opts, + ), + [1], + ], + [ + 'decr', + async () => + target.decr( + { key: 'root-decr', pathAndAmountMap: { '[0]': 1 } }, + opts, + ), + [-1], + ], + [ + 'update', + async () => + target.update( + { + key: 'root-update', + pathAndValueMap: { '[0]': 'zero' }, + }, + opts, + ), + ['zero'], + ], + [ + 'add', + async () => + target.add( + { + key: 'root-add', + pathAndValueMap: { '[0]': 'zero' }, + }, + opts, + ), + [['zero']], + ], + ]; + + it.each(rootArrayOperations)( + '%s initializes a fresh root array for [0]', + async (_method, run, expected) => { + expect((await run()).res).toEqual(expected); + }, + ); + + it('removes a root array index', async () => { + await target.set({ key: 'root-remove', value: ['zero'] }, opts); + const result = await target.remove( + { key: 'root-remove', paths: ['[0]'] }, + opts, + ); + expect(result.res).toEqual([]); + }); + + it('supports indexes, mixed paths, repeated indexes, and quoted keys', async () => { + await target.set( + { + key: 'path-shapes', + value: { + a: [{ count: 1, items: [] }], + some: { path: [{}, {}] }, + nested: [[0, 1]], + 'a.b': [{ 'c.d': {} }], + "quote'and\\slash": {}, + }, + }, + opts, + ); + + await target.incr( + { key: 'path-shapes', pathAndAmountMap: { 'a[0].count': 2 } }, + opts, + ); + await target.decr( + { key: 'path-shapes', pathAndAmountMap: { 'a[0].count': 1 } }, + opts, + ); + await target.add( + { key: 'path-shapes', pathAndValueMap: { 'a[0].items': 'x' } }, + opts, + ); + await target.update( + { + key: 'path-shapes', + pathAndValueMap: { + 'some.path[1].to.value': 'mixed', + 'nested[0][1]': 'repeated', + '["a.b"][0]["c.d"].value': 'dotted', + "['quote\\'and\\\\slash'].value": 'escaped', + }, + }, + opts, + ); + const removed = await target.remove( + { key: 'path-shapes', paths: ['a[0].items[0]'] }, + opts, + ); + + expect(removed.res).toEqual({ + a: [{ count: 2, items: [] }], + some: { path: [{}, { to: { value: 'mixed' } }] }, + nested: [[0, 'repeated']], + 'a.b': [{ 'c.d': { value: 'dotted' } }], + "quote'and\\slash": { value: 'escaped' }, + }); + }); + + it( + 'keeps distinct aliases for colliding attribute names in one operation', + async () => { + const result = await target.update( + { + key: 'alias-collision', + pathAndValueMap: { 'a-b': 1, ab: 2 }, + }, + opts, + ); + expect(result.res).toEqual({ 'a-b': 1, ab: 2 }); + }, + ); + + it('keeps legacy empty dot chunks while parsing array paths', async () => { + const result = await target.update( + { + key: 'empty-dot-chunks', + pathAndValueMap: { '.a..b.': 1 }, + }, + opts, + ); + expect(result.res).toEqual({ a: { b: 1 } }); + }); + + it('rejects paths with conflicting map and list parents in one operation', async () => { + await expect( + target.update( + { + key: 'conflicting-parent', + pathAndValueMap: { 'a.b': 1, 'a[0]': 2 }, + }, + opts, + ), + ).rejects.toMatchObject({ statusCode: 400 }); + }); + + it('does not create indexed ancestors or sparse arrays', async () => { + await target.set({ key: 'indexed-ancestor', value: [{}] }, opts); + await expect( + target.update( + { + key: 'indexed-ancestor', + pathAndValueMap: { '[4].x': 1 }, + }, + opts, + ), + ).rejects.toMatchObject({ name: 'ValidationException' }); + expect( + (await target.get({ key: 'indexed-ancestor' }, opts)).res, + ).toEqual([{}]); + }); + + it.each([ + '[', + 'a[', + 'a[0', + 'a[]', + 'a[-1]', + 'a[1.5]', + 'a["unterminated]', + "a['unterminated]", + 'a["x"', + 'a[0]tail', + ])('rejects malformed path %s', async (path) => { + await expect( + target.update( + { key: 'bad-path', pathAndValueMap: { [path]: 1 } }, + opts, + ), + ).rejects.toMatchObject({ statusCode: 400 }); + }); + + it.each(['__proto__', 'constructor', 'prototype'])( + 'rejects unsafe quoted path key %s in every path method', + async (unsafeKey) => { + const path = `["${unsafeKey}"]`; + await expect( + target.incr( + { key: 'unsafe-incr', pathAndAmountMap: { [path]: 1 } }, + opts, + ), + ).rejects.toMatchObject({ statusCode: 400 }); + await expect( + target.decr( + { key: 'unsafe-decr', pathAndAmountMap: { [path]: 1 } }, + opts, + ), + ).rejects.toMatchObject({ statusCode: 400 }); + await expect( + target.update( + { key: 'unsafe-update', pathAndValueMap: { [path]: 1 } }, + opts, + ), + ).rejects.toMatchObject({ statusCode: 400 }); + await expect( + target.add( + { key: 'unsafe-add', pathAndValueMap: { [path]: 1 } }, + opts, + ), + ).rejects.toMatchObject({ statusCode: 400 }); + await expect( + target.remove( + { key: 'unsafe-remove', paths: [path] }, + opts, + ), + ).rejects.toMatchObject({ statusCode: 400 }); + }, + ); + }); + describe('prototype-chain safety', () => { const unsafeSegments = ['__proto__', 'constructor', 'prototype']; diff --git a/src/backend/stores/systemKv/SystemKVStore.ts b/src/backend/stores/systemKv/SystemKVStore.ts index e3ce1ee98..830bb6e14 100644 --- a/src/backend/stores/systemKv/SystemKVStore.ts +++ b/src/backend/stores/systemKv/SystemKVStore.ts @@ -18,7 +18,6 @@ */ import { metrics } from '@opentelemetry/api'; -import { PuterStore } from '../types'; import type { KvMutation } from '../../clients/event/types'; import type { Actor } from '../../core/actor'; import { @@ -26,10 +25,6 @@ import { SYSTEM_ACTOR, SYSTEM_ACTOR_UUID, } from '../../core/actor'; -import { - PUTER_KV_STORE_TABLE_DEFINITION, - PUTER_KV_STORE_TABLE_NAME, -} from './tableDefinition'; import { HttpError } from '../../core/http'; import { decodeCursor, @@ -37,17 +32,22 @@ import { normalizeLimit, normalizeOffset, } from '../../util/pagination'; +import { PuterStore } from '../types'; import { cacheTtlSecondsFor, decodeCachedRead, encodeCachedHit, encodeCachedMiss, - kvCacheKey, KV_CACHE_BLOCK_MARKER, + kvCacheKey, resolveKvCacheSettings, type KvCachedItem, type KvCacheSettings, } from './readCache'; +import { + PUTER_KV_STORE_TABLE_DEFINITION, + PUTER_KV_STORE_TABLE_NAME, +} from './tableDefinition'; const meter = metrics.getMeter('puter-backend'); @@ -229,7 +229,7 @@ const getNamespace = (actor: Actor, opts?: KVOpts): string => { actor.effectiveApp?.uid ?? opts?.appUuid ?? KV_GLOBAL_APP_KEY; - return kvNamespace(actor.user.uuid, appUuid); + return kvNamespace(actor.user.uuid!, appUuid); }; /** The namespace one user's data for one app lives in. */ @@ -285,25 +285,100 @@ const unsafeKeyError = (key: string, subject: string): HttpError => legacyCode: 'bad_request', }); -/** - * Reject a caller-supplied document path (`a.b.c`, optionally with `[0]` list - * indexes) whose segments would walk onto the prototype chain. - */ -const assertPath = (valPath: string): void => { +type PathToken = + | { type: 'key'; value: string } + | { type: 'index'; value: number }; + +const invalidPathError = (): HttpError => + new HttpError(400, 'kv: path has invalid syntax', { + legacyCode: 'bad_request', + }); + +/** Parse the dot and bracket forms accepted by the KV document methods. */ +const parsePath = (valPath: string): PathToken[] => { if (typeof valPath !== 'string') throw new HttpError(400, 'kv: path must be a string', { legacyCode: 'bad_request', }); - for (const chunk of valPath.split('.')) { - const name = chunk.split(/\[\d*\]/g)[0]; - if (UNSAFE_OBJECT_KEYS.has(name)) - throw unsafeKeyError(name, 'path segment'); + if (valPath === '') return []; + + const tokens: PathToken[] = []; + let position = 0; + let expectSegment = true; + while (position < valPath.length) { + if (valPath[position] === '.') { + while (valPath[position] === '.') position++; + expectSegment = true; + continue; + } + + if (!expectSegment && valPath[position] !== '[') + throw invalidPathError(); + + if (valPath[position] === '[') { + position++; + if (position >= valPath.length) throw invalidPathError(); + const quote = valPath[position]; + if (quote === '"' || quote === "'") { + position++; + let value = ''; + let closed = false; + while (position < valPath.length) { + const char = valPath[position++]; + if (char === '\\') { + if (position >= valPath.length) + throw invalidPathError(); + value += valPath[position++]; + } else if (char === quote) { + closed = true; + break; + } else { + value += char; + } + } + if (!closed || valPath[position] !== ']') + throw invalidPathError(); + position++; + if (UNSAFE_OBJECT_KEYS.has(value)) + throw unsafeKeyError(value, 'path segment'); + tokens.push({ type: 'key', value }); + } else { + const start = position; + while ( + position < valPath.length && + /[0-9]/.test(valPath[position]) + ) + position++; + if (start === position || valPath[position] !== ']') + throw invalidPathError(); + const value = Number(valPath.slice(start, position)); + if (!Number.isSafeInteger(value)) throw invalidPathError(); + position++; + tokens.push({ type: 'index', value }); + } + expectSegment = false; + continue; + } + + if (!expectSegment) throw invalidPathError(); + const start = position; + while ( + position < valPath.length && + valPath[position] !== '.' && + valPath[position] !== '[' + ) + position++; + const value = valPath.slice(start, position); + if (!value) throw invalidPathError(); + if (UNSAFE_OBJECT_KEYS.has(value)) + throw unsafeKeyError(value, 'path segment'); + tokens.push({ type: 'key', value }); + expectSegment = false; } + return tokens; }; -const assertPaths = (paths: string[]): void => { - for (const valPath of paths) assertPath(valPath); -}; +const parsePaths = (paths: string[]): PathToken[][] => paths.map(parsePath); const isOversizedExpression = (err: Error): boolean => /expression size/i.test(err.message); @@ -366,6 +441,12 @@ const isPlainObject = (value: unknown): value is Record => const objectsEqual = (left: unknown, right: unknown): boolean => { if (left === right) return true; + if (Array.isArray(left) && Array.isArray(right)) { + return ( + left.length === right.length && + left.every((value, index) => objectsEqual(value, right[index])) + ); + } if (!isPlainObject(left) || !isPlainObject(right)) return false; const leftKeys = Object.keys(left); const rightKeys = Object.keys(right); @@ -377,15 +458,36 @@ const objectsEqual = (left: unknown, right: unknown): boolean => { return true; }; -const cleanAttrName = (chunk: string): string => - `#${chunk.replaceAll(PATH_CLEANER_REGEX, '')}`; +class PathExpressionRenderer { + readonly names: Record = { '#value': 'value' }; + #aliases = new Map(); + + path(tokens: PathToken[]): string { + let result = '#value'; + for (const token of tokens) { + if (token.type === 'index') result += `[${token.value}]`; + else result += `.${this.#alias(token.value)}`; + } + return result; + } + + #alias(key: string): string { + let alias = this.#aliases.get(key); + if (alias) return alias; + alias = `#p${this.#aliases.size}_${key.replaceAll(PATH_CLEANER_REGEX, '')}`; + this.#aliases.set(key, alias); + this.names[alias] = key; + return alias; + } +} /** The `SET` assignment `incr` renders for one path. */ -const incrSetStatement = (valPath: string, idx: number): string => { - const attrName = ['value', ...valPath.split('.')] - .filter(Boolean) - .map(cleanAttrName) - .join('.'); +const incrSetStatement = ( + tokens: PathToken[], + idx: number, + renderer: PathExpressionRenderer, +): string => { + const attrName = renderer.path(tokens); return `${attrName} = if_not_exists(${attrName}, :start${idx}) + :incr${idx}`; }; @@ -401,8 +503,14 @@ const incrSetStatement = (valPath: string, idx: number): string => { export const INCR_EXPRESSION_BUDGET_BYTES = 3584; /** Size of the update expression `incr` would send for `paths`. */ -export const incrExpressionBytes = (paths: string[]): number => - Buffer.byteLength(`SET ${paths.map(incrSetStatement).join(', ')}`); +export const incrExpressionBytes = (paths: string[]): number => { + const renderer = new PathExpressionRenderer(); + return Buffer.byteLength( + `SET ${parsePaths(paths) + .map((tokens, index) => incrSetStatement(tokens, index, renderer)) + .join(', ')}`, + ); +}; /** * Split paths into batches whose expressions each fit `maxBytes`, preserving @@ -1315,6 +1423,7 @@ export class SystemKVStore extends PuterStore { offset, includeTotal, fetchUntilFull, + reverse, }: { as?: 'keys' | 'values' | 'entries'; limit?: number; @@ -1323,6 +1432,7 @@ export class SystemKVStore extends PuterStore { offset?: number; includeTotal?: boolean; fetchUntilFull?: boolean; + reverse?: boolean; }, opts?: KVOpts, ): Promise< @@ -1348,7 +1458,50 @@ export class SystemKVStore extends PuterStore { cap: MAX_LIST_OFFSET, label: 'kv: offset', }); - const pageKey = decodeCursor(cursor, 'kv: cursor'); + if (reverse !== undefined && typeof reverse !== 'boolean') { + throw new HttpError(400, 'kv: reverse must be a boolean', { + legacyCode: 'bad_request', + }); + } + const decodedCursor = decodeCursor(cursor, 'kv: cursor'); + let pageKey = decodedCursor; + let cursorReverse = false; + if (decodedCursor !== undefined) { + if ( + !decodedCursor || + typeof decodedCursor !== 'object' || + Array.isArray(decodedCursor) + ) { + throw new HttpError(400, 'invalid kv: cursor', { + legacyCode: 'bad_request', + }); + } + if (Object.hasOwn(decodedCursor, 'reverse')) { + if ( + decodedCursor.reverse !== true || + !decodedCursor.key || + typeof decodedCursor.key !== 'object' || + Array.isArray(decodedCursor.key) || + Object.keys(decodedCursor.key).length === 0 + ) { + throw new HttpError(400, 'invalid kv: cursor', { + legacyCode: 'bad_request', + }); + } + cursorReverse = true; + pageKey = decodedCursor.key as Record; + } + if (reverse !== undefined && reverse !== cursorReverse) { + throw new HttpError( + 400, + 'kv: reverse conflicts with cursor direction', + { + legacyCode: 'bad_request', + }, + ); + } + } + const effectiveReverse = reverse ?? cursorReverse; const normalizedPattern = normalizePattern(pattern); if (pageKey !== undefined && normalizedOffset !== undefined) { @@ -1397,6 +1550,7 @@ export class SystemKVStore extends PuterStore { '', false, { + scanIndexForward: !effectiveReverse, ...(normalizedPattern ? { beginsWith: { @@ -1497,7 +1651,11 @@ export class SystemKVStore extends PuterStore { } while (countKey); } - const nextCursor = encodeCursor(nextKey); + const nextCursor = encodeCursor( + nextKey && effectiveReverse + ? { reverse: true, key: nextKey } + : nextKey, + ); return { res: { items, @@ -1608,15 +1766,16 @@ export class SystemKVStore extends PuterStore { { legacyCode: 'bad_request' }, ); } - assertPaths(Object.keys(pathAndAmountMap)); + const pathTokens = parsePaths(Object.keys(pathAndAmountMap)); const actor = ensureActor(opts); const namespace = getNamespace(actor, opts); const probeUsage = await this.#assertNotPrivate(namespace, key, opts); - const setStatements = Object.keys(pathAndAmountMap).map( - (valPath, idx) => incrSetStatement(valPath, idx), + const renderer = new PathExpressionRenderer(); + const setStatements = pathTokens.map((tokens, idx) => + incrSetStatement(tokens, idx, renderer), ); const valueAttributeValues = Object.entries(pathAndAmountMap).reduce( (acc, [_path, amt], idx) => { @@ -1626,18 +1785,6 @@ export class SystemKVStore extends PuterStore { }, {} as Record, ); - const valueAttributeNames = Object.entries(pathAndAmountMap).reduce( - (acc, [valPath]) => { - ['value', ...valPath.split('.')] - .filter(Boolean) - .forEach((chunk) => { - const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; - acc[cleanAttrName(cleanedChunk)] = cleanedChunk; - }); - return acc; - }, - {} as Record, - ); // Fold the TTL into the same UpdateItem so a counter bump is a single // write instead of incr + a separate expireAt. if_not_exists keeps the @@ -1651,18 +1798,17 @@ export class SystemKVStore extends PuterStore { }); setStatements.push('#ttl = if_not_exists(#ttl, :ttl)'); valueAttributeValues[':ttl'] = ttlSeconds; - valueAttributeNames['#ttl'] = 'ttl'; + renderer.names['#ttl'] = 'ttl'; } const updateExpression = `SET ${setStatements.join(', ')}`; - const expressionNames = { ...valueAttributeNames, '#value': 'value' }; const runUpdate = () => this.clients.dynamo.update( this.tableName, { key, namespace }, updateExpression, valueAttributeValues, - expressionNames, + renderer.names, ); // Most increments land on an item whose parent maps already exist (a @@ -1687,7 +1833,7 @@ export class SystemKVStore extends PuterStore { createPathsUsage = await this.createPaths( namespace, key, - Object.keys(pathAndAmountMap), + pathTokens, ); response = await runUpdate(); } @@ -1732,7 +1878,7 @@ export class SystemKVStore extends PuterStore { for (const val of Object.values(pathAndValueMap)) { assertValue(val); } - assertPaths(Object.keys(pathAndValueMap)); + const pathTokens = parsePaths(Object.keys(pathAndValueMap)); const actor = ensureActor(opts); const namespace = getNamespace(actor, opts); @@ -1742,18 +1888,14 @@ export class SystemKVStore extends PuterStore { const createPathsUsage = await this.createPaths( namespace, key, - Object.keys(pathAndValueMap), + pathTokens, ); - const setStatements = Object.entries(pathAndValueMap).map( - ([valPath], idx) => { - const attrName = ['value', ...valPath.split('.')] - .filter(Boolean) - .map(cleanAttrName) - .join('.'); - return `${attrName} = list_append(if_not_exists(${attrName}, :emptyList${idx}), :append${idx})`; - }, - ); + const renderer = new PathExpressionRenderer(); + const setStatements = pathTokens.map((tokens, idx) => { + const attrName = renderer.path(tokens); + return `${attrName} = list_append(if_not_exists(${attrName}, :emptyList${idx}), :append${idx})`; + }); const valueAttributeValues = Object.entries(pathAndValueMap).reduce( (acc, [_path, val], idx) => { acc[`:append${idx}`] = Array.isArray(val) ? val : [val]; @@ -1762,25 +1904,12 @@ export class SystemKVStore extends PuterStore { }, {} as Record, ); - const valueAttributeNames = Object.entries(pathAndValueMap).reduce( - (acc, [valPath]) => { - ['value', ...valPath.split('.')] - .filter(Boolean) - .forEach((chunk) => { - const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; - acc[cleanAttrName(cleanedChunk)] = cleanedChunk; - }); - return acc; - }, - {} as Record, - ); - const response = await this.clients.dynamo.update( this.tableName, { key, namespace }, `SET ${setStatements.join(', ')}`, valueAttributeValues, - { ...valueAttributeNames, '#value': 'value' }, + renderer.names, ); await this.#committed(actor, namespace, [key], 'set'); @@ -1805,34 +1934,16 @@ export class SystemKVStore extends PuterStore { legacyCode: 'bad_request', }); } - assertPaths(paths); + const pathTokens = parsePaths(paths); const actor = ensureActor(opts); const namespace = getNamespace(actor, opts); const probeUsage = await this.#assertNotPrivate(namespace, key, opts); - const removeStatements = paths.map((valPath) => { - return ['value', ...valPath.split('.')] - .filter(Boolean) - .map((chunk) => { - const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; - const indexSuffix = chunk.slice(cleanedChunk.length); - return `${cleanAttrName(cleanedChunk)}${indexSuffix}`; - }) - .join('.'); - }); - const valueAttributeNames = paths.reduce( - (acc, valPath) => { - ['value', ...valPath.split('.')] - .filter(Boolean) - .forEach((chunk) => { - const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; - acc[cleanAttrName(cleanedChunk)] = cleanedChunk; - }); - return acc; - }, - {} as Record, + const renderer = new PathExpressionRenderer(); + const removeStatements = pathTokens.map((tokens) => + renderer.path(tokens), ); try { @@ -1841,7 +1952,7 @@ export class SystemKVStore extends PuterStore { { key, namespace }, `REMOVE ${removeStatements.join(', ')}`, undefined, - { ...valueAttributeNames, '#value': 'value' }, + renderer.names, ); await this.#committed(actor, namespace, [key], 'set'); return { @@ -1896,7 +2007,7 @@ export class SystemKVStore extends PuterStore { for (const val of Object.values(pathAndValueMap)) { assertValue(val); } - assertPaths(Object.keys(pathAndValueMap)); + const pathTokens = parsePaths(Object.keys(pathAndValueMap)); const actor = ensureActor(opts); const namespace = getNamespace(actor, opts); @@ -1905,18 +2016,14 @@ export class SystemKVStore extends PuterStore { const createPathsUsage = await this.createPaths( namespace, key, - Object.keys(pathAndValueMap), + pathTokens, ); - const setStatements = Object.entries(pathAndValueMap).map( - ([valPath], idx) => { - const attrName = ['value', ...valPath.split('.')] - .filter(Boolean) - .map(cleanAttrName) - .join('.'); - return `${attrName} = :value${idx}`; - }, - ); + const renderer = new PathExpressionRenderer(); + const setStatements = pathTokens.map((tokens, idx) => { + const attrName = renderer.path(tokens); + return `${attrName} = :value${idx}`; + }); const valueAttributeValues = Object.entries(pathAndValueMap).reduce( (acc, [_path, val], idx) => { acc[`:value${idx}`] = val; @@ -1924,19 +2031,6 @@ export class SystemKVStore extends PuterStore { }, {} as Record, ); - const valueAttributeNames = Object.entries(pathAndValueMap).reduce( - (acc, [valPath]) => { - ['value', ...valPath.split('.')] - .filter(Boolean) - .forEach((chunk) => { - const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; - acc[cleanAttrName(cleanedChunk)] = cleanedChunk; - }); - return acc; - }, - {} as Record, - ); - if (ttl !== undefined) { const ttlSeconds = Number(ttl); if (Number.isNaN(ttlSeconds)) @@ -1946,7 +2040,7 @@ export class SystemKVStore extends PuterStore { const timestamp = Math.floor(Date.now() / 1000) + ttlSeconds; setStatements.push('#ttl = :ttl'); valueAttributeValues[':ttl'] = timestamp; - valueAttributeNames['#ttl'] = 'ttl'; + renderer.names['#ttl'] = 'ttl'; } const response = await this.clients.dynamo.update( @@ -1954,7 +2048,7 @@ export class SystemKVStore extends PuterStore { { key, namespace }, `SET ${setStatements.join(', ')}`, valueAttributeValues, - { ...valueAttributeNames, '#value': 'value' }, + renderer.names, ); await this.#committed(actor, namespace, [key], 'set'); @@ -2034,37 +2128,53 @@ export class SystemKVStore extends PuterStore { } /** - * Ensure each intermediate map layer exists for a set of nested paths. - * Returns write units consumed. DDB can't set nested paths on missing - * parents in one expression, so we walk the layers and `SET ... - * if_not_exists(..., {})` each one. + * Create missing parent containers one layer at a time and return write + * units consumed. Indexed ancestors must already exist. */ private async createPaths( namespace: string, key: string, - pathList: string[], + pathList: PathToken[][], ): Promise { - assertPaths(pathList); - const nestedMapValue = (() => { + const rootIsList = pathList[0]?.[0]?.type === 'index'; + if ( + pathList.some( + (tokens) => + tokens[0] && + (tokens[0].type === 'index') !== rootIsList, + ) + ) + throw new HttpError( + 400, + 'kv: paths require incompatible roots', + { + legacyCode: 'bad_request', + }, + ); + if (rootIsList) return [] as unknown[]; + const valueRoot: Record = {}; let hasPaths = false; - pathList.forEach((valPath) => { - if (!valPath) return; + pathList.forEach((tokens) => { + if (tokens.length === 0) return; hasPaths = true; - const chunks = valPath.split('.').filter(Boolean); let cursor: Record = valueRoot; - for (let i = 0; i < chunks.length - 1; i++) { - const chunk = chunks[i]; + for (let i = 0; i < tokens.length - 1; i++) { + const token = tokens[i]; + if (token.type === 'index') break; + const next = tokens[i + 1]; + const container = next.type === 'index' ? [] : {}; // Own properties only: an inherited hit here would mean // walking (and then writing to) the prototype chain. - const existing = Object.hasOwn(cursor, chunk) - ? cursor[chunk] + const existing = Object.hasOwn(cursor, token.value) + ? cursor[token.value] : undefined; if (!isPlainObject(existing)) { - cursor[chunk] = {}; + cursor[token.value] = container; } - cursor = cursor[chunk] as Record; + if (Array.isArray(container)) break; + cursor = cursor[token.value] as Record; } }); return hasPaths ? valueRoot : null; @@ -2072,39 +2182,49 @@ export class SystemKVStore extends PuterStore { if (!nestedMapValue) return 0; - const allIntermediatePaths = new Set(); - pathList.forEach((valPath) => { - const chunks = ['value', ...valPath.split('.')].filter(Boolean); - for (let i = 1; i < chunks.length; i++) { - allIntermediatePaths.add(chunks.slice(0, i).join('.')); + const allIntermediatePaths = new Map(); + const containerTypes = new Map(); + allIntermediatePaths.set('', []); + for (const tokens of pathList) { + for (let i = 1; i < tokens.length; i++) { + const prefix = tokens.slice(0, i); + if (prefix.at(-1)?.type === 'index') continue; + const id = JSON.stringify(prefix); + allIntermediatePaths.set(id, prefix); + const containerType = tokens[i].type; + const existingType = containerTypes.get(id); + if (existingType && existingType !== containerType) { + throw new HttpError( + 400, + 'kv: paths require incompatible containers', + { legacyCode: 'bad_request' }, + ); + } + containerTypes.set(id, containerType); } - }); + } let writeUnits = 0; - const orderedPaths = [...allIntermediatePaths].sort( - (left, right) => left.split('.').length - right.split('.').length, + const orderedPaths = [...allIntermediatePaths.values()].sort( + (left, right) => left.length - right.length, ); for (const layerPath of orderedPaths) { - const chunks = layerPath.split('.'); - const attrName = chunks.map(cleanAttrName).join('.'); - const expressionNames: Record = {}; - chunks.forEach((chunk) => { - const cleanedChunk = chunk.split(/\[\d*\]/g)[0]; - expressionNames[cleanAttrName(cleanedChunk)] = cleanedChunk; - }); - const isRootLayer = layerPath === 'value'; + const renderer = new PathExpressionRenderer(); + const attrName = renderer.path(layerPath); + const isRootLayer = layerPath.length === 0; + const nextType = containerTypes.get(JSON.stringify(layerPath)); const expressionValues = isRootLayer ? { ':nestedMap': nestedMapValue } - : { ':emptyMap': {} }; - const valueToken = isRootLayer ? ':nestedMap' : ':emptyMap'; + : { ':emptyContainer': nextType === 'index' ? [] : {} }; + const valueToken = isRootLayer ? ':nestedMap' : ':emptyContainer'; const response = await this.clients.dynamo.update( this.tableName, { key, namespace }, `SET ${attrName} = if_not_exists(${attrName}, ${valueToken})`, expressionValues, - expressionNames, + renderer.names, ); writeUnits += Number(response.ConsumedCapacity?.CapacityUnits ?? 0); diff --git a/src/docs/src/KV.md b/src/docs/src/KV.md index a51e716ab..68432453b 100644 --- a/src/docs/src/KV.md +++ b/src/docs/src/KV.md @@ -223,7 +223,7 @@ These Key-Value Store features are supported out of the box when using Puter.js: - **[`puter.kv.del()`](/KV/del/)** - Delete a key-value pair - **[`puter.kv.expire()`](/KV/expire/)** - Set key expiration in seconds - **[`puter.kv.expireAt()`](/KV/expireAt/)** - Set key expiration timestamp -- **[`puter.kv.list()`](/KV/list/)** - List all keys +- **[`puter.kv.list()`](/KV/list/)** - List keys in ascending or descending order - **[`puter.kv.flush()`](/KV/flush/)** - Clear all data ## Examples diff --git a/src/docs/src/KV/add.md b/src/docs/src/KV/add.md index a19a97297..2b16a1032 100644 --- a/src/docs/src/KV/add.md +++ b/src/docs/src/KV/add.md @@ -27,10 +27,12 @@ An array is appended element by element, so wrap a single value in an array to a #### `pathAndValue` (Object) (optional) -An object where each key is a dot-separated path (for example, `"profile.tags"`) and each value is the value (or values) to add at that path. +An object where each key is a path (for example, `"profile.tags"`) and each value is the value (or values) to add at that path. Appended values follow the same limits as [`puter.kv.set()`](/KV/set/): **400 KB**, and every number within **±9,007,199,254,740,991** — a larger one is stored clamped to that bound. +Paths support dot notation, array indexes at any level (`[0]`, `items[0]`, or `some.path[1].to.value`), and quoted property names (`["key.with.dots"]`). An empty path (`""`) targets the whole stored value. Use non-negative integer indexes in brackets to address arrays. When a path continues through an array element (for example, `[0].score`), that element must already exist. Missing object parents are created automatically; sparse array elements are not created. + ## Return value Returns a `Promise` that resolves to the updated value stored at `key`. diff --git a/src/docs/src/KV/decr.md b/src/docs/src/KV/decr.md index 25507b72a..a04182867 100755 --- a/src/docs/src/KV/decr.md +++ b/src/docs/src/KV/decr.md @@ -29,6 +29,8 @@ When `amount` is an object: Decrements a property within an object value stored - Key: the path to the property (e.g., `"user.score"`) - Value: the amount to decrement by +Paths support dot notation, array indexes at any level (`[0]`, `items[0]`, or `some.path[1].to.value`), and quoted property names (`["key.with.dots"]`). An empty path (`""`) targets the whole stored value. Use non-negative integer indexes in brackets to address arrays. When a path continues through an array element (for example, `[0].score`), that element must already exist. Missing object parents are created automatically; sparse array elements are not created. + ## Return Value Returns the new value of the key after the decrement operation. diff --git a/src/docs/src/KV/incr.md b/src/docs/src/KV/incr.md index 5fe1ca0d9..94ca57c3e 100755 --- a/src/docs/src/KV/incr.md +++ b/src/docs/src/KV/incr.md @@ -31,6 +31,8 @@ When `amount` is an object: Increments a property within an object value stored `amount` must be within **±9,007,199,254,740,991** (`Number.MAX_SAFE_INTEGER`); a larger one is applied clamped to that bound. A counter stays exact only while its total is inside the same range — store anything that has to count past it as a string with [`puter.kv.set()`](/KV/set/). +Paths support dot notation, array indexes at any level (`[0]`, `items[0]`, or `some.path[1].to.value`), and quoted property names (`["key.with.dots"]`). An empty path (`""`) targets the whole stored value. Use non-negative integer indexes in brackets to address arrays. When a path continues through an array element (for example, `[0].score`), that element must already exist. Missing object parents are created automatically; sparse array elements are not created. + ## Return Value Returns the new value of the key after the increment operation. diff --git a/src/docs/src/KV/list.md b/src/docs/src/KV/list.md index d4b113fa4..1debc7053 100755 --- a/src/docs/src/KV/list.md +++ b/src/docs/src/KV/list.md @@ -6,7 +6,7 @@ platforms: [websites, apps, nodejs, workers] Returns an array of all keys in the user's key-value store for the current app. If the user has no keys, the array will be empty. -Results are sorted lexicographically (string order) by key. +Results are sorted lexicographically (string order) by key, ascending by default. Pass `reverse: true` to list keys in descending order. ## Syntax @@ -36,8 +36,9 @@ An object with the following optional properties: - `pattern` (String): Same as the `pattern` parameter. - `returnValues` (Boolean): Same as the `returnValues` parameter. +- `reverse` (Boolean): Lists keys in descending order when `true`. Defaults to `false`. Works with full listings, pagination, and streams; by itself, it keeps the plain-array return shape. - `limit` (Number): Maximum number of items to return in a single call. -- `cursor` (String): A pagination cursor from a previous call. Pass the `cursor` value returned by the previous page to fetch the next one. +- `cursor` (String): A pagination cursor from a previous call. Pass the `cursor` value returned by the previous page to fetch the next one. The cursor preserves the listing direction; omit `reverse` to keep it, or pass the same value. A conflicting direction is rejected. - `offset` (Number): Skips the given number of items before the page starts. Not recommended — requests get slower and more expensive the larger the offset; prefer `cursor`. Maximum `5000`, and cannot be combined with `cursor`. - `includeTotal` (Boolean): If `true`, the result includes a `total` count of every item matching the query (across all pages). The count is metered and its cost grows with the size of your store — request it once (on the first page) and avoid it in hot paths. If you only need to know whether more pages exist, check for `cursor` instead of counting. - `fetchUntilFull` (Boolean): A page can come back with fewer than `limit` items even when more exist (for example when expired keys are excluded). If `true`, the page is filled up to `limit` items when possible. Requires `limit`. @@ -69,6 +70,24 @@ for await (const page of puter.kv.list({ pattern: 'log:*', stream: true })) { ## Examples +List keys in reverse order + +```html;kv-list-reverse + + + + + + +``` + Retrieve all keys in the user's key-value store for the current app ```html;kv-list diff --git a/src/docs/src/KV/remove.md b/src/docs/src/KV/remove.md index 928c56d62..89c7ebfad 100644 --- a/src/docs/src/KV/remove.md +++ b/src/docs/src/KV/remove.md @@ -4,7 +4,7 @@ description: Remove values at one or more paths from a key in the user's own key platforms: [websites, apps, nodejs, workers] --- -Remove values from an existing key by path. Paths use dot notation to target nested fields. +Remove values from an existing key by path. Paths can target nested fields and array elements. ## Syntax @@ -20,7 +20,9 @@ The key to remove values from. #### `paths` (String[]) (required) -One or more dot-separated paths to remove (for example, `"profile.bio"`). +One or more paths to remove (for example, `"profile.bio"`). + +Paths support dot notation, array indexes at any level (`[0]`, `items[0]`, or `some.path[1].to.value`), and quoted property names (`["key.with.dots"]`). An empty path (`""`) targets the whole stored value. Use non-negative integer indexes in brackets to address arrays. Removing an array element shifts later elements down by one index. ## Return value diff --git a/src/docs/src/KV/update.md b/src/docs/src/KV/update.md index 485614a39..aa949cd15 100644 --- a/src/docs/src/KV/update.md +++ b/src/docs/src/KV/update.md @@ -22,7 +22,7 @@ The key to update. #### `pathAndValueMap` (Object) (required) -An object where each key is a dot-separated path (for example, `"profile.name"`) and each value is the new value for that path. +An object where each key is a path (for example, `"profile.name"`) and each value is the new value for that path. Each value follows the same limits as [`puter.kv.set()`](/KV/set/): **400 KB**, and every number within **±9,007,199,254,740,991** — a larger one is stored clamped to that bound. @@ -30,12 +30,31 @@ Each value follows the same limits as [`puter.kv.set()`](/KV/set/): **400 KB**, Time-to-live for the key, in seconds. +Paths support dot notation, array indexes at any level (`[0]`, `items[0]`, or `some.path[1].to.value`), and quoted property names (`["key.with.dots"]`). An empty path (`""`) targets the whole stored value. Use non-negative integer indexes in brackets to address arrays. When a path continues through an array element (for example, `[0].score`), that element must already exist. Missing object parents are created automatically; sparse array elements are not created. + ## Return value Returns a `Promise` that resolves to the updated value stored at `key`. ## Examples +Update an element of a root array + +```html;kv-update-root-array + + + + + + +``` + Update nested fields and refresh the TTL ```html;kv-update diff --git a/src/puter-js/src/modules/kv/list.js b/src/puter-js/src/modules/kv/list.js index 4fcb52e38..73a1fab88 100644 --- a/src/puter-js/src/modules/kv/list.js +++ b/src/puter-js/src/modules/kv/list.js @@ -116,6 +116,7 @@ const normalizeListPattern = (pattern) => { */ /** * Lists keys in the store for the current app, sorted lexicographically. + * Set `reverse: true` for descending key order. * Returns just the keys, `KVPair` objects when `returnValues` is `true`, or * a `KVListPage` when any pagination option (`limit`, `cursor`, `offset`, * `includeTotal`, `fetchUntilFull`) is used. With `stream: true` it instead @@ -158,6 +159,12 @@ export function list (patternOrOptions, returnValuesOrOptConfig, maybeOptConfig) if ( isOptionsObject ) { const input = patternOrOptions; + if ( input.reverse !== undefined ) { + if ( typeof input.reverse !== 'boolean' ) { + throw { message: 'reverse must be a boolean', code: 'invalid_request' }; + } + options.reverse = input.reverse; + } if ( typeof input.pattern === 'string' ) { pattern = input.pattern; } diff --git a/src/puter-js/src/modules/kv/types.js b/src/puter-js/src/modules/kv/types.js index 13a522c16..a7052f991 100644 --- a/src/puter-js/src/modules/kv/types.js +++ b/src/puter-js/src/modules/kv/types.js @@ -77,6 +77,8 @@ * Options object form of the arguments to `list()`. * * @typedef {Object} KVListOptions + * @property {boolean} [reverse=false] Return keys in descending order. A cursor preserves its + * direction when omitted; an explicitly conflicting direction is rejected. * @property {string} [pattern] Prefix-based key filter. A trailing `*` is a wildcard; both `abc` and * `abc*` match keys starting with `abc`. Defaults to `*`, matching all keys. * @property {boolean} [returnValues] When `true`, results contain `KVPair` objects with `key` and diff --git a/src/puter-js/tests/api/suites/kv.suite.ts b/src/puter-js/tests/api/suites/kv.suite.ts index d3489840e..4c7ad3223 100644 --- a/src/puter-js/tests/api/suites/kv.suite.ts +++ b/src/puter-js/tests/api/suites/kv.suite.ts @@ -256,6 +256,26 @@ export default suite('kv', { t.assert.ok(keys.includes('kv-suite-all-2')); }, + 'list reverse supports arrays, pages, cursor resume, and streams': async (t) => { + await t.assert.rejects(async () => t.puter.kv.list({ reverse: 'true' } as never)); + const pattern = 'kv-suite-reverse-*'; + const keys = ['kv-suite-reverse-a', 'kv-suite-reverse-b', 'kv-suite-reverse-c']; + for (let i = 0; i < keys.length; i++) await t.puter.kv.set(keys[i], i); + const descending = [...keys].reverse(); + t.assert.deepEqual(await t.puter.kv.list({ pattern, reverse: true }), descending); + t.assert.deepEqual(await t.puter.kv.list({ pattern, reverse: false }), keys); + const first = await t.puter.kv.list({ pattern, reverse: true, returnValues: true, limit: 1 }); + t.assert.deepEqual(first.items, [{ key: keys[2], value: 2 }]); + const rest = await t.puter.kv.list({ pattern, cursor: first.cursor, limit: 3 }); + t.assert.deepEqual(rest.items, [keys[1], keys[0]]); + await t.assert.rejects(() => t.puter.kv.list({ pattern, cursor: first.cursor, reverse: false, limit: 1 })); + const streamed: string[] = []; + for await (const page of t.puter.kv.list({ pattern, reverse: true, stream: true, limit: 1 })) { + streamed.push(...page.items); + } + t.assert.deepEqual(streamed, descending); + }, + 'list returns keys in lexicographic order': async (t) => { await t.puter.kv.set('kv-suite-sorted-c', 1); await t.puter.kv.set('kv-suite-sorted-a', 1);