From e75ccb0a41ddb2c8cbaf31c8b78465eaf6c433ee Mon Sep 17 00:00:00 2001 From: Daniel Salazar Date: Tue, 24 Mar 2026 19:18:42 -0700 Subject: [PATCH] feat: root level kv accesses, and installed app listing + server health check fix (#2719) * feat: root level kv accesses, and installed app listing * fix: revert server health check --- extensions/extensionController/package.json | 1 - .../src/ExtensionController.ts | 13 +- extensions/installedApps/package.json | 23 ++ .../controllers/InstalledAppsController.ts | 73 +++++ extensions/installedApps/src/index.ts | 6 + extensions/installedApps/tsconfig.json | 28 ++ src/backend/src/helpers.js | 2 +- .../ServerHealthService.js | 82 ++++-- .../kvstore/KVStoreInterfaceService.js | 14 +- .../src/routers/auth/revoke-user-app.js | 12 +- .../DynamoKVStore/DynamoKVStore.test.ts | 63 ++++ .../services/DynamoKVStore/DynamoKVStore.ts | 184 +++++++----- .../src/services/PermissionAPIService.js | 7 +- .../OpenAIVideoGenerationProvider.ts | 246 ++++++++++++++++ .../OpenAiVideoGenerationProvider/models.ts | 59 ++++ .../src/services/auth/PermissionService.js | 169 +++-------- src/puter-js/src/modules/KV.js | 276 ++++++++++++++++-- src/puter-js/test/kv.test.js | 72 +++++ src/puter-js/types/modules/kv.d.ts | 37 ++- tests/puterJsApiTests/kv.test.ts | 48 +++ 20 files changed, 1116 insertions(+), 299 deletions(-) create mode 100644 extensions/installedApps/package.json create mode 100644 extensions/installedApps/src/controllers/InstalledAppsController.ts create mode 100644 extensions/installedApps/src/index.ts create mode 100644 extensions/installedApps/tsconfig.json create mode 100644 src/backend/src/services/ai/video/providers/OpenAiVideoGenerationProvider/OpenAIVideoGenerationProvider.ts create mode 100644 src/backend/src/services/ai/video/providers/OpenAiVideoGenerationProvider/models.ts diff --git a/extensions/extensionController/package.json b/extensions/extensionController/package.json index 0f5a2f7cf..7035aa372 100644 --- a/extensions/extensionController/package.json +++ b/extensions/extensionController/package.json @@ -11,7 +11,6 @@ "author": "", "license": "ISC", "devDependencies": { - "@types/express": "^4.17.21", "@types/node": "^24.9.1", "ts-node": "^10.9.2", "typescript": "^5.9.3" diff --git a/extensions/extensionController/src/ExtensionController.ts b/extensions/extensionController/src/ExtensionController.ts index 3f1506b4d..a1317c0e8 100644 --- a/extensions/extensionController/src/ExtensionController.ts +++ b/extensions/extensionController/src/ExtensionController.ts @@ -43,19 +43,14 @@ const createMethodDecorator = (method: HttpMethod) => { adminUsernames?: string[], ) => { const { allowedAppIds, ...options } = routeOptions ?? {}; - return < - P extends Record = Record< - string, - string | undefined - >, - >( - target: RequestHandler

, + return ( + target: RequestHandler, _context: ClassMethodDecoratorContext< This, ( this: This, - ...args: Parameters> - ) => ReturnType> + ...args: Parameters> + ) => ReturnType> >, ) => { _context.addInitializer(function () { diff --git a/extensions/installedApps/package.json b/extensions/installedApps/package.json new file mode 100644 index 000000000..0f5a2f7cf --- /dev/null +++ b/extensions/installedApps/package.json @@ -0,0 +1,23 @@ +{ + "name": "@puter/extension-controller", + "version": "1.0.0", + "description": "", + "main": "src/index.js", + "type": "module", + "scripts": { + "postinstall": "tsc --noCheck" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@types/express": "^4.17.21", + "@types/node": "^24.9.1", + "ts-node": "^10.9.2", + "typescript": "^5.9.3" + }, + "dependencies": { + "http-status-codes": "^2.3.0", + "stripe": "^19.1.0" + } +} \ No newline at end of file diff --git a/extensions/installedApps/src/controllers/InstalledAppsController.ts b/extensions/installedApps/src/controllers/InstalledAppsController.ts new file mode 100644 index 000000000..cb38abb62 --- /dev/null +++ b/extensions/installedApps/src/controllers/InstalledAppsController.ts @@ -0,0 +1,73 @@ +import type { BaseDatabaseAccessService } from '@heyputer/backend/src/services/database/BaseDatabaseAccessService.js'; +import { Request, Response } from 'express'; +import '../../../api.d.ts'; + +const { Controller, Get, ExtensionController, HttpError } = extension.import('extensionController'); + +const getAppIconUrl = extension.import('core').util.helpers.get_app_icon_url; + +@Controller('/installedApps') +export class InstalledAppsController extends ExtensionController { + + static ALLOWED_ORDER_BY = ['id', + 'name', + 'uid', + 'title', + 'owner_id', + 'dt']; + #db: BaseDatabaseAccessService; + constructor (db: BaseDatabaseAccessService) { + super(); + this.#db = db; + } + + @Get('/', { subdomain: 'api' }) + async getInstalledApps (req: Request, res: Response): Promise { + const actor = req.actor; + if ( ! actor ) { + throw Error('actor not found in context'); + } + if ( actor.type.app ) { + throw new HttpError(403, 'Apps are not allowed to access this resource'); + } + if ( ! InstalledAppsController.ALLOWED_ORDER_BY.includes(req.query.orderBy) ) { + throw new HttpError(400, `Invalid orderBy field. Allowed fields are: ${InstalledAppsController.ALLOWED_ORDER_BY.join(', ')}`); + } + + const page = Math.min(req.query.page || 1, 1); + const limit = Math.min(Math.max(req.query.limit || 100, 100), 0); + const offset = (page - 1) * limit; + + const installedApps = await this.#db.read( + `SELECT + apps.id, + apps.name, + apps.uid, + apps.title, + apps.description, + apps.owner_id, + MIN(perm.dt) as installed_at, + MAX(app_opens.ts) as last_opened + FROM apps + LEFT JOIN user_to_app_permissions as perm ON apps.id = perm.app_id + LEFT JOIN app_opens ON app_opens.app_uid = apps.uid AND app_opens.user_id = ? + WHERE perm.user_id = ? + GROUP BY apps.id, apps.name, apps.uid, apps.title, apps.description, apps.icon, apps.owner_id + ORDER BY apps.${req.query.orderBy || 'created_at'} ${req.query.desc ? 'DESC' : 'ASC'} + LIMIT ? + OFFSET ?`, + [actor.uid, actor.uid, limit, offset], + ) as { + id: number; + name: string; + uid: string; + title: string; + description: string; + owner_id: number; + installed_at: Date; + last_opened: Date | null; + }[]; + + res.send(installedApps.map((app) => ({ ...app, iconUrl: getAppIconUrl(app) }))); + } +} diff --git a/extensions/installedApps/src/index.ts b/extensions/installedApps/src/index.ts new file mode 100644 index 000000000..359a28882 --- /dev/null +++ b/extensions/installedApps/src/index.ts @@ -0,0 +1,6 @@ +const allowedOrderBy = ['id', + 'name', + 'uid', + 'title', + 'owner_id', + 'dt']; \ No newline at end of file diff --git a/extensions/installedApps/tsconfig.json b/extensions/installedApps/tsconfig.json new file mode 100644 index 000000000..c9cbd48a9 --- /dev/null +++ b/extensions/installedApps/tsconfig.json @@ -0,0 +1,28 @@ +{ + "compilerOptions": { + "target": "ES2024", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true, + "sourceMap": true, + "noEmitOnError": true, + "noImplicitAny": false, + "allowJs": true, + "checkJs": false, + }, + "include": [ + "./**/*.ts", + "./**/*.d.ts" + ], + "exclude": [ + "**/*.test.ts", + "**/*.spec.ts", + "**/test/**", + "**/tests/**", + "node_modules", + "dist", + "*.js" + ] +} \ No newline at end of file diff --git a/src/backend/src/helpers.js b/src/backend/src/helpers.js index 037e7ca28..55e979292 100644 --- a/src/backend/src/helpers.js +++ b/src/backend/src/helpers.js @@ -480,7 +480,7 @@ export async function get_app (options) { return app; } -const get_app_icon_url = (app, size) => { +export const get_app_icon_url = (app, size) => { const iconIsBase64 = isBase64AppIcon(app); const svc_appIcon = servicesContainer.services.get('app-icon'); const app_uid = app.uid ?? app.uuid; diff --git a/src/backend/src/modules/core/ServerHealthService/ServerHealthService.js b/src/backend/src/modules/core/ServerHealthService/ServerHealthService.js index e76a18533..e71435f3b 100644 --- a/src/backend/src/modules/core/ServerHealthService/ServerHealthService.js +++ b/src/backend/src/modules/core/ServerHealthService/ServerHealthService.js @@ -16,11 +16,10 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -import { asyncSafeSetInterval, TeePromise } from '@heyputer/putility/src/libs/promise.js'; -import { BaseService } from '../../../services/BaseService.js'; -import { kv } from '../../../util/kvSingleton.js'; -import { ServerHealthRedisCacheKeys } from './ServerHealthRedisCacheKeys.js'; - +const { ServerHealthRedisCacheKeys } = require('./ServerHealthRedisCacheKeys.js'); +const BaseService = require('../../../services/BaseService'); +const { kv } = require('../../../util/kvSingleton'); +const { promise } = require('@heyputer/putility').libs; const SECOND = 1000; /** @@ -34,14 +33,24 @@ const SECOND = 1000; * This service is designed to work primarily on Linux systems, reading system metrics * from `/proc/meminfo` and handling alarms via an external 'alarm' service. */ -export class ServerHealthService extends BaseService { +class ServerHealthService extends BaseService { + static USE = { + linuxutil: 'core.util.linuxutil', + }; - #checks = []; - #failures = []; - #stats = {}; + static MODULES = { + fs: require('fs'), + }; + + _construct () { + this.checks_ = []; + this.failures_ = []; + } async _init () { - this.#initServiceChecks(); + this.init_service_checks_(); + + this.stats_ = {}; } /** @@ -52,26 +61,45 @@ export class ServerHealthService extends BaseService { * @param {none} - This method does not take any parameters. * @returns {void} - This method does not return any value. */ - #initServiceChecks () { + init_service_checks_ () { const svc_alarm = this.services.get('alarm'); - asyncSafeSetInterval(async () => { + /** + * Initializes periodic health checks for the server. + * + * This method sets up an interval to run all registered health checks + * at a specified frequency. It manages the execution of checks, handles + * timeouts, and logs errors or triggers alarms when checks fail. + * + * @private + * @method init_service_checks_ + * @memberof ServerHealthService + * @param {none} - No parameters are passed to this method. + * @returns {void} + */ + promise.asyncSafeSetInterval(async () => { + this.log.tick('service checks'); const check_failures = []; - for ( const { name, fn, chainable } of this.#checks ) { - const p_timeout = new TeePromise(); + for ( const { name, fn, chainable } of this.checks_ ) { + const p_timeout = new promise.TeePromise(); + /** + * Creates a TeePromise to handle potential timeouts during health checks. + * + * @returns {Promise} A promise that can be resolved or rejected from multiple places. + */ const timeout = setTimeout(() => { p_timeout.reject(new Error('Health check timed out')); }, 5 * SECOND); - try { await Promise.race([ fn(), p_timeout, ]); + clearTimeout(timeout); } catch ( err ) { // Trigger an alarm if this check isn't already in the failure list - if ( this.#failures.some(v => v.name === name) ) { - continue; + if ( this.failures_.some(v => v.name === name) ) { + return; } svc_alarm.create( @@ -81,22 +109,20 @@ export class ServerHealthService extends BaseService { ); check_failures.push({ name }); - console.error(`Error for healthcheck fail on ${name}: ${ err.stack}`); + this.log.error(`Error for healthcheck fail on ${name}: ${ err.stack}`); // Run the on_fail handlers for ( const fn of chainable.on_fail_ ) { try { await fn(err); } catch ( e ) { - console.error(`Error in on_fail handler for ${name}`, e); + this.log.error(`Error in on_fail handler for ${name}`, e); } } - } finally { - clearTimeout(timeout); } } - this.#failures = check_failures; + this.failures_ = check_failures; }, 10 * SECOND, null, { onBehindSchedule: (drift) => { svc_alarm.create( @@ -116,7 +142,7 @@ export class ServerHealthService extends BaseService { * direct manipulation of the service's data. */ async get_stats () { - return { ...this.#stats }; + return { ...this.stats_ }; } add_check (name, fn) { @@ -127,7 +153,7 @@ export class ServerHealthService extends BaseService { return chainable; }, }; - this.#checks.push({ name, fn, chainable }); + this.checks_.push({ name, fn, chainable }); return chainable; } @@ -153,15 +179,19 @@ export class ServerHealthService extends BaseService { } // Compute status - const failures = this.#failures.map(v => v.name); + const failures = this.failures_.map(v => v.name); const status = { ok: failures.length === 0, ...(failures.length ? { failed: failures } : {}), }; // Cache with 5 second TTL - await kv.set(cacheKey, JSON.stringify(status), { EX: 5 }); + await kv.set(cacheKey, JSON.stringify(status), { + EX: 5, + }); return status; } } + +module.exports = { ServerHealthService }; diff --git a/src/backend/src/modules/kvstore/KVStoreInterfaceService.js b/src/backend/src/modules/kvstore/KVStoreInterfaceService.js index a10a8a402..9a55246ba 100644 --- a/src/backend/src/modules/kvstore/KVStoreInterfaceService.js +++ b/src/backend/src/modules/kvstore/KVStoreInterfaceService.js @@ -99,6 +99,7 @@ class KVStoreInterfaceService extends BaseService { description: 'Get a value by key.', parameters: { key: { type: 'json', required: true }, + optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, }, result: { type: 'json' }, }, @@ -108,6 +109,8 @@ class KVStoreInterfaceService extends BaseService { key: { type: 'string', required: true }, value: { type: 'json' }, expireAt: { type: 'number' }, + optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, + }, result: { type: 'void' }, }, @@ -115,6 +118,7 @@ class KVStoreInterfaceService extends BaseService { description: 'Delete a value by key.', parameters: { key: { type: 'string' }, + optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, }, result: { type: 'void' }, }, @@ -133,12 +137,13 @@ class KVStoreInterfaceService extends BaseService { cursor: { type: 'string', }, + optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, }, result: { type: 'json' }, }, flush: { description: 'Delete all key-value pairs.', - parameters: {}, + parameters: { optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' } }, result: { type: 'void' }, }, update: { @@ -147,6 +152,7 @@ class KVStoreInterfaceService extends BaseService { key: { type: 'string', required: true }, pathAndValueMap: { type: 'json', required: true, description: 'map of period-joined path to value' }, ttl: { type: 'number', description: 'optional TTL in seconds for the whole object' }, + optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, }, result: { type: 'json', description: 'The updated value' }, }, @@ -155,6 +161,7 @@ class KVStoreInterfaceService extends BaseService { parameters: { key: { type: 'string', required: true }, pathAndValueMap: { type: 'json', required: true, description: 'map of period-joined path to value to append' }, + optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, }, result: { type: 'json', description: 'The updated value' }, }, @@ -163,6 +170,7 @@ class KVStoreInterfaceService extends BaseService { parameters: { key: { type: 'string', required: true }, paths: { type: 'json', required: true, description: 'list of period-joined paths to remove' }, + optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, }, result: { type: 'json', description: 'The updated value' }, }, @@ -171,6 +179,7 @@ class KVStoreInterfaceService extends BaseService { parameters: { key: { type: 'string', required: true }, pathAndAmountMap: { type: 'json', required: true, description: 'map of period-joined path to amount to increment by' }, + optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, }, result: { type: 'json', description: 'The updated value' }, }, @@ -179,6 +188,7 @@ class KVStoreInterfaceService extends BaseService { parameters: { key: { type: 'string', required: true }, pathAndAmountMap: { type: 'json', required: true, description: 'map of period-joined path to amount to increment by' }, + optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, }, result: { type: 'json', description: 'The updated value' }, @@ -188,6 +198,7 @@ class KVStoreInterfaceService extends BaseService { parameters: { key: { type: 'string', required: true }, timestamp: { type: 'number', required: true }, + optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, }, result: { type: 'number' }, @@ -197,6 +208,7 @@ class KVStoreInterfaceService extends BaseService { parameters: { key: { type: 'string', required: true }, ttl: { type: 'number', required: true }, + optConfig: { type: 'json', description: 'additional options for get, e.g. { appUuid: "someId" }' }, }, result: { type: 'number' }, diff --git a/src/backend/src/routers/auth/revoke-user-app.js b/src/backend/src/routers/auth/revoke-user-app.js index 7a4b58880..fd455f8ef 100644 --- a/src/backend/src/routers/auth/revoke-user-app.js +++ b/src/backend/src/routers/auth/revoke-user-app.js @@ -16,16 +16,16 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -const eggspress = require('../../api/eggspress'); -const { UserActorType } = require('../../services/auth/Actor'); -const { Context } = require('../../util/context'); -const APIError = require('../../api/APIError'); +import eggspress from '../../api/eggspress.js'; +import { UserActorType } from '../../services/auth/Actor.js'; +import { Context } from '../../util/context.js'; +import { APIError } from '../../api/APIError.js'; -module.exports = eggspress('/auth/revoke-user-app', { +export const revokeUserAppPermsRoute = eggspress('/auth/revoke-user-app', { subdomain: 'api', auth2: true, allowedMethods: ['POST'], -}, async (req, res, next) => { +}, async (req, res) => { const x = Context.get(); const svc_permission = x.get('services').get('permission'); diff --git a/src/backend/src/services/DynamoKVStore/DynamoKVStore.test.ts b/src/backend/src/services/DynamoKVStore/DynamoKVStore.test.ts index 9d3515857..a6ba42c10 100644 --- a/src/backend/src/services/DynamoKVStore/DynamoKVStore.test.ts +++ b/src/backend/src/services/DynamoKVStore/DynamoKVStore.test.ts @@ -479,6 +479,69 @@ describe('DynamoKVStore', async () => { expect((stored as { a?: { b?: number[] } }).a?.b).toEqual([1, 5]); }); + it('supports appUuid namespace isolation for non-app actors', async () => { + const actor = makeActor(22); + const key = 'override-key'; + const optConfig = { appUuid: 'override-app-a' }; + + await su.sudo(actor, () => kvStore.set({ key, value: 'default-value' })); + await su.sudo(actor, () => kvStore.set({ key, value: 'override-value', optConfig })); + + const defaultRead = await su.sudo(actor, () => kvStore.get({ key })); + const overrideRead = await su.sudo(actor, () => kvStore.get({ key, optConfig })); + const overrideList = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: `${key}*`, optConfig })) as string[]; + + expect(defaultRead).toBe('default-value'); + expect(overrideRead).toBe('override-value'); + expect(overrideList).toContain(key); + + await su.sudo(actor, () => kvStore.del({ key, optConfig })); + const afterOverrideDelete = await su.sudo(actor, () => kvStore.get({ key, optConfig })); + const defaultAfterOverrideDelete = await su.sudo(actor, () => kvStore.get({ key })); + + expect(afterOverrideDelete).toBeNull(); + expect(defaultAfterOverrideDelete).toBe('default-value'); + }); + + it('flush with appUuid only clears that override namespace', async () => { + const actor = makeActor(23); + const overrideA = { appUuid: 'flush-override-a' }; + const overrideB = { appUuid: 'flush-override-b' }; + const keyA = 'flush-override-key-a'; + const keyB = 'flush-override-key-b'; + const defaultKey = 'flush-default-key'; + + await su.sudo(actor, () => kvStore.set({ key: keyA, value: 'A', optConfig: overrideA })); + await su.sudo(actor, () => kvStore.set({ key: keyB, value: 'B', optConfig: overrideB })); + await su.sudo(actor, () => kvStore.set({ key: defaultKey, value: 'default' })); + + await su.sudo(actor, () => kvStore.flush({ optConfig: overrideA })); + + expect(await su.sudo(actor, () => kvStore.get({ key: keyA, optConfig: overrideA }))).toBeNull(); + expect(await su.sudo(actor, () => kvStore.get({ key: keyB, optConfig: overrideB }))).toBe('B'); + expect(await su.sudo(actor, () => kvStore.get({ key: defaultKey }))).toBe('default'); + }); + + it('ignores appUuid when actor already has an app context', async () => { + const appActor = makeActor(24, 'real-app'); + const userActor = makeActor(24); + const key = 'app-context-override-ignore'; + + await su.sudo(appActor, () => kvStore.set({ + key, + value: 'from-app-context', + optConfig: { appUuid: 'fake-app' }, + })); + + const appRead = await su.sudo(appActor, () => kvStore.get({ key })); + const userReadFromFake = await su.sudo(userActor, () => kvStore.get({ key, optConfig: { appUuid: 'fake-app' } })); + const userReadFromReal = await su.sudo(userActor, () => kvStore.get({ key, optConfig: { appUuid: 'real-app' } })); + + expect(appRead).toBe('from-app-context'); + expect(userReadFromFake).toBeNull(); + expect(userReadFromReal).toBe('from-app-context'); + }); + it('enforces key and value size limits', async () => { const actor = makeActor(11); const oversizedKey = 'a'.repeat(((config as unknown as Record).kv_max_key_size as number) + 1); diff --git a/src/backend/src/services/DynamoKVStore/DynamoKVStore.ts b/src/backend/src/services/DynamoKVStore/DynamoKVStore.ts index b338e426f..18daa0a78 100644 --- a/src/backend/src/services/DynamoKVStore/DynamoKVStore.ts +++ b/src/backend/src/services/DynamoKVStore/DynamoKVStore.ts @@ -33,21 +33,21 @@ export class DynamoKVStore { await this.#ddbClient.createTableIfNotExists({ ...PUTER_KV_STORE_TABLE_DEFINITION, TableName: this.#tableName }, 'ttl'); } - #getNameSpace (actor: Actor) { + #getNameSpace (actor: Actor, appUuidOverride?: string) { if ( actor.type instanceof SystemActorType ) { return 'v1:system'; } else { - const app = actor.type?.app ?? undefined; + const appUuid = !actor.type?.app ? (appUuidOverride || undefined) : actor.type.app.uid; const user = actor.type?.user ?? undefined; if ( ! user ) throw new Error('User not found'); - return `v1:${app ? `${user.uuid}:${app.uid}` + return `v1:${appUuid ? `${user.uuid}:${appUuid}` : `${user.uuid}:${this.#enableMigrationFromSQL ? DynamoKVStore.LEGACY_GLOBAL_APP_KEY : DynamoKVStore.GLOBAL_APP_KEY}`}`; } } @Span('kv:get') - async get ({ key }: { key: string | string[]; }): Promise { + async get ({ key, optConfig }: { key: string | string[]; optConfig?: { appUuid?: string } }): Promise { if ( key === '' ) { throw APIError.create('field_empty', null, { key: 'key', @@ -55,10 +55,10 @@ export class DynamoKVStore { } const actor = Context.get('actor'); - const app = actor.type?.app ?? undefined; + const appUuid = !actor.type?.app ? optConfig?.appUuid : actor.type.app.uid; const user = actor.type?.user ?? undefined; - const namespace = this.#getNameSpace(actor); + const namespace = this.#getNameSpace(actor, optConfig?.appUuid); const multi = Array.isArray(key); const keys = multi ? key : [key]; @@ -92,15 +92,19 @@ export class DynamoKVStore { if ( this.#enableMigrationFromSQL ) { const key_hash = murmurhash.v3(key); - const kv_row = await this.#sqlClient.read('SELECT * FROM kv WHERE user_id=? AND app=? AND kkey_hash=? LIMIT 1', - [user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY, key_hash]); + const kv_row = await this.#sqlClient.read( + 'SELECT * FROM kv WHERE user_id=? AND app=? AND kkey_hash=? LIMIT 1', + [user.id, appUuid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY, key_hash], + ); if ( kv_row[0]?.value ) { // update and delete from this table (async () => { await this.set({ key: kv_row[0].key, value: kv_row[0].value }); - await this.#sqlClient.write('DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?', - [user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY, key_hash]); + await this.#sqlClient.write( + 'DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?', + [user.id, appUuid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY, key_hash], + ); })(); values.push(kv_row[0]?.value); continue; @@ -141,7 +145,7 @@ export class DynamoKVStore { } @Span('kv:set') - async set ({ key, value, expireAt }: { key: string; value: unknown; expireAt?: number; }): Promise { + async set ({ key, value, expireAt, optConfig }: { key: string; value: unknown; expireAt?: number; optConfig?: { appUuid?: string } }): Promise { const context = Context.get(); const actor = context.get('actor'); @@ -161,7 +165,7 @@ export class DynamoKVStore { this.get({ key }); } - const namespace = this.#getNameSpace(actor); + const namespace = this.#getNameSpace(actor, optConfig?.appUuid); const res = await this.#ddbClient.put(this.#tableName, { namespace, @@ -175,14 +179,14 @@ export class DynamoKVStore { } @Span('kv:del') - async del ({ key }: { key: string; }): Promise { + async del ({ key, optConfig}: { key: string;optConfig?: { appUuid?: string } }): Promise { const actor = Context.get('actor'); const app = actor.type?.app ?? undefined; const user = actor.type?.user ?? undefined; if ( ! user ) throw new Error('User not found'); - const namespace = this.#getNameSpace(actor); + const namespace = this.#getNameSpace(actor, optConfig?.appUuid); const res = await this.#ddbClient.del(this.#tableName, { namespace, @@ -193,8 +197,10 @@ export class DynamoKVStore { if ( this.#enableMigrationFromSQL ) { const key_hash = murmurhash.v3(key); - await this.#sqlClient.write('DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?', - [user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY, key_hash]); + await this.#sqlClient.write( + 'DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?', + [user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY, key_hash], + ); } return true; @@ -277,11 +283,13 @@ export class DynamoKVStore { limit, cursor, pattern, + optConfig, }: { as?: 'keys' | 'values' | 'entries'; limit?: number; cursor?: string | Record; pattern?: string; + optConfig?: { appUuid?: string } }): Promise< | string[] | unknown[] @@ -296,20 +304,22 @@ export class DynamoKVStore { const user = actor.type?.user ?? undefined; if ( ! user ) throw new Error('User not found'); - const namespace = this.#getNameSpace(actor); + const namespace = this.#getNameSpace(actor, optConfig?.appUuid); const normalizedLimit = this.#normalizeLimit(limit); const pageKey = this.#decodeCursor(cursor); const normalizedPattern = this.#normalizePattern(pattern); const paginated = normalizedLimit !== undefined || pageKey !== undefined; - const entriesRes = await this.#ddbClient.query(this.#tableName, - { namespace }, - normalizedLimit ?? 0, - pageKey, - '', - false, - normalizedPattern ? { beginsWith: { key: 'key', value: normalizedPattern } } : undefined); + const entriesRes = await this.#ddbClient.query( + this.#tableName, + { namespace }, + normalizedLimit ?? 0, + pageKey, + '', + false, + normalizedPattern ? { beginsWith: { key: 'key', value: normalizedPattern } } : undefined, + ); this.#meteringService.incrementUsage(actor, 'kv:read', entriesRes.ConsumedCapacity?.CapacityUnits ?? 1); @@ -326,8 +336,10 @@ export class DynamoKVStore { }); if ( this.#enableMigrationFromSQL && !paginated ) { - const oldEntries = await this.#sqlClient.read('SELECT * FROM kv WHERE user_id=? AND app=?', - [user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY]); + const oldEntries = await this.#sqlClient.read( + 'SELECT * FROM kv WHERE user_id=? AND app=?', + [user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY], + ); oldEntries.forEach(oldEntry => { if ( normalizedPattern && !oldEntry.kkey?.startsWith(normalizedPattern) ) { return; @@ -370,18 +382,20 @@ export class DynamoKVStore { } @Span('kv:flush') - async flush () { + async flush ({ optConfig}: { optConfig?: { appUuid?: string } }) { const actor = Context.get('actor'); const app = actor.type.app ?? undefined; const user = actor.type?.user ?? undefined; if ( ! user ) throw new Error('User not found'); - const namespace = this.#getNameSpace(actor); + const namespace = this.#getNameSpace(actor, optConfig?.appUuid); // Query all keys - const entriesRes = await this.#ddbClient.query(this.#tableName, - { namespace }); + const entriesRes = await this.#ddbClient.query( + this.#tableName, + { namespace }, + ); const entries = entriesRes.Items ?? []; const readUsage = entriesRes?.ConsumedCapacity?.CapacityUnits ?? 0; @@ -406,15 +420,17 @@ export class DynamoKVStore { this.#meteringService.incrementUsage(actor, 'kv:write', writeUsage); if ( this.#enableMigrationFromSQL ) { - await this.#sqlClient.write('DELETE FROM kv WHERE user_id=? AND app=?', - [user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY]); + await this.#sqlClient.write( + 'DELETE FROM kv WHERE user_id=? AND app=?', + [user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY], + ); } return !!allRes; } @Span('kv:expireAt') - async expireAt ({ key, timestamp }: { key: string; timestamp: number; }): Promise { + async expireAt ({ key, timestamp, optConfig}: { key: string; timestamp: number;optConfig?: { appUuid?: string } }): Promise { if ( key === '' ) { throw APIError.create('field_empty', null, { key: 'key', @@ -423,11 +439,11 @@ export class DynamoKVStore { timestamp = Number(timestamp); - return await this.#expireAt(key, timestamp); + return await this.#expireAt(key, timestamp, optConfig); } @Span('kv:expire') - async expire ({ key, ttl }: { key: string; ttl: number; }): Promise { + async expire ({ key, ttl, optConfig}: { key: string; ttl: number; optConfig?: { appUuid?: string } }): Promise { if ( key === '' ) { throw APIError.create('field_empty', null, { key: 'key', @@ -439,7 +455,7 @@ export class DynamoKVStore { // timestamp in seconds let timestamp = Math.floor(Date.now() / 1000) + ttl; - return await this.#expireAt(key, timestamp); + return await this.#expireAt(key, timestamp, optConfig); } async #createPaths ( namespace: string, key: string, pathList: string[]) { @@ -515,11 +531,13 @@ export class DynamoKVStore { : { ':emptyMap': {} }; const valueToken = isRootLayer ? ':nestedMap' : ':emptyMap'; // Issue update to set layer to {} if not exists - const layerUpsertRes = await this.#ddbClient.update(this.#tableName, - { key, namespace }, - `SET ${attrName} = if_not_exists(${attrName}, ${valueToken})`, - expressionValues, - expressionNames); + const layerUpsertRes = await this.#ddbClient.update( + this.#tableName, + { key, namespace }, + `SET ${attrName} = if_not_exists(${attrName}, ${valueToken})`, + expressionValues, + expressionNames, + ); writeUnits += layerUpsertRes.ConsumedCapacity?.CapacityUnits ?? 0; if ( isRootLayer && objectsEqual(layerUpsertRes.Attributes?.value, nestedMapValue) ) { return writeUnits; @@ -530,7 +548,7 @@ export class DynamoKVStore { // Ideally the paths support syntax like "a.b[2].c" @Span('kv:incr') - async incr>({ key, pathAndAmountMap }: { key: string; pathAndAmountMap: T; }): Promise> { + async incr>({ key, pathAndAmountMap, optConfig }: { key: string; pathAndAmountMap: T;optConfig?: { appUuid?: string } }): Promise> { if ( Object.values(pathAndAmountMap).find((v) => typeof v !== 'number') ) { throw new Error('All values in pathAndAmountMap must be numbers'); } @@ -549,7 +567,7 @@ export class DynamoKVStore { const user = actor.type?.user ?? undefined; if ( ! user ) throw new Error('User not found'); - const namespace = this.#getNameSpace(actor); + const namespace = this.#getNameSpace(actor, optConfig?.appUuid); if ( this.#enableMigrationFromSQL ) { // trigger get to move element if exists @@ -579,23 +597,25 @@ export class DynamoKVStore { return acc; }, {} as Record); - const res = await this.#ddbClient.update(this.#tableName, - { key, namespace }, - `SET ${[...setStatements].join(', ')}`, - valueAttributeValues, - { ...valueAttributeNames, '#value': 'value' }); + const res = await this.#ddbClient.update( + this.#tableName, + { key, namespace }, + `SET ${[...setStatements].join(', ')}`, + valueAttributeValues, + { ...valueAttributeNames, '#value': 'value' }, + ); writeUnits += res.ConsumedCapacity?.CapacityUnits ?? 0; this.#meteringService.incrementUsage(actor, 'kv:write', writeUnits); return res.Attributes?.value; } - async decr>({ key, pathAndAmountMap }: { key: string; pathAndAmountMap: T; }) { - return await this.incr({ key, pathAndAmountMap: Object.fromEntries(Object.entries(pathAndAmountMap).map(([k, v]) => [k, -v])) as T }); + async decr>({ key, pathAndAmountMap, optConfig }: { key: string; pathAndAmountMap: T; optConfig?: { appUuid?: string } }) { + return await this.incr({ key, pathAndAmountMap: Object.fromEntries(Object.entries(pathAndAmountMap).map(([k, v]) => [k, -v])) as T, optConfig }); } @Span('kv:add') - async add ({ key, pathAndValueMap }: { key: string; pathAndValueMap: Record; }): Promise { + async add ({ key, pathAndValueMap, optConfig}: { key: string; pathAndValueMap: Record; optConfig?: { appUuid?: string } }): Promise { if ( !pathAndValueMap || Object.keys(pathAndValueMap).length === 0 ) { throw new Error('invalid use of #add: no pathAndValueMap'); } @@ -610,7 +630,7 @@ export class DynamoKVStore { const user = actor.type?.user ?? undefined; if ( ! user ) throw new Error('User not found'); - const namespace = this.#getNameSpace(actor); + const namespace = this.#getNameSpace(actor, optConfig?.appUuid); if ( this.#enableMigrationFromSQL ) { // trigger get to move element if exists @@ -640,11 +660,13 @@ export class DynamoKVStore { return acc; }, {} as Record); - const res = await this.#ddbClient.update(this.#tableName, - { key, namespace }, - `SET ${[...setStatements].join(', ')}`, - valueAttributeValues, - { ...valueAttributeNames, '#value': 'value' }); + const res = await this.#ddbClient.update( + this.#tableName, + { key, namespace }, + `SET ${[...setStatements].join(', ')}`, + valueAttributeValues, + { ...valueAttributeNames, '#value': 'value' }, + ); writeUnits += res.ConsumedCapacity?.CapacityUnits ?? 0; this.#meteringService.incrementUsage(actor, 'kv:write', writeUnits); @@ -652,7 +674,7 @@ export class DynamoKVStore { } @Span('kv:remove') - async remove ({ key, paths }: { key: string; paths: string[]; }): Promise { + async remove ({ key, paths, optConfig }: { key: string; paths: string[]; optConfig?: { appUuid?: string } }): Promise { if ( !paths || paths.length === 0 ) { throw new Error('invalid use of #remove: no paths'); } @@ -667,7 +689,7 @@ export class DynamoKVStore { const user = actor.type?.user ?? undefined; if ( ! user ) throw new Error('User not found'); - const namespace = this.#getNameSpace(actor); + const namespace = this.#getNameSpace(actor, optConfig?.appUuid); if ( this.#enableMigrationFromSQL ) { // trigger get to move element if exists @@ -695,11 +717,13 @@ export class DynamoKVStore { }, {} as Record); try { - const res = await this.#ddbClient.update(this.#tableName, - { key, namespace }, - `REMOVE ${removeStatements.join(', ')}`, - undefined, - { ...valueAttributeNames, '#value': 'value' }); + const res = await this.#ddbClient.update( + this.#tableName, + { key, namespace }, + `REMOVE ${removeStatements.join(', ')}`, + undefined, + { ...valueAttributeNames, '#value': 'value' }, + ); this.#meteringService.incrementUsage(actor, 'kv:write', res?.ConsumedCapacity?.CapacityUnits ?? 1); return res.Attributes?.value; @@ -714,7 +738,7 @@ export class DynamoKVStore { } @Span('kv:update') - async update ({ key, pathAndValueMap, ttl }: { key: string; pathAndValueMap: Record; ttl?: number; }): Promise { + async update ({ key, pathAndValueMap, ttl, optConfig }: { key: string; pathAndValueMap: Record; ttl?: number; optConfig?: { appUuid?: string } }): Promise { if ( !pathAndValueMap || Object.keys(pathAndValueMap).length === 0 ) { throw new Error('invalid use of #update: no pathAndValueMap'); } @@ -729,7 +753,7 @@ export class DynamoKVStore { const user = actor.type?.user ?? undefined; if ( ! user ) throw new Error('User not found'); - const namespace = this.#getNameSpace(actor); + const namespace = this.#getNameSpace(actor, optConfig?.appUuid); if ( this.#enableMigrationFromSQL ) { // trigger get to move element if exists @@ -769,36 +793,40 @@ export class DynamoKVStore { valueAttributeNames['#ttl'] = 'ttl'; } - const res = await this.#ddbClient.update(this.#tableName, - { key, namespace }, - `SET ${[...setStatements].join(', ')}`, - valueAttributeValues, - { ...valueAttributeNames, '#value': 'value' }); + const res = await this.#ddbClient.update( + this.#tableName, + { key, namespace }, + `SET ${[...setStatements].join(', ')}`, + valueAttributeValues, + { ...valueAttributeNames, '#value': 'value' }, + ); writeUnits += res.ConsumedCapacity?.CapacityUnits ?? 0; this.#meteringService.incrementUsage(actor, 'kv:write', writeUnits); return res.Attributes?.value; } - async #expireAt (key: string, timestamp: number) { + async #expireAt (key: string, timestamp: number, optConfig?: { appUuid?: string }) { const actor = Context.get('actor'); const user = actor.type?.user ?? undefined; if ( ! user ) throw new Error('User not found'); - const namespace = this.#getNameSpace(actor); + const namespace = this.#getNameSpace(actor, optConfig?.appUuid); // if possibly migrating from old SQL store, get entry first to move to dynamo if ( this.#enableMigrationFromSQL ) { await this.get({ key }); } - const res = await this.#ddbClient.update(this.#tableName, - { key, namespace }, - 'SET #ttl = :ttl, #value = if_not_exists(#value, :defaultValue)', - { ':ttl': timestamp, ':defaultValue': null }, - { '#ttl': 'ttl', '#value': 'value' }); + const res = await this.#ddbClient.update( + this.#tableName, + { key, namespace }, + 'SET #ttl = :ttl, #value = if_not_exists(#value, :defaultValue)', + { ':ttl': timestamp, ':defaultValue': null }, + { '#ttl': 'ttl', '#value': 'value' }, + ); // meter usage this.#meteringService.incrementUsage(actor, 'kv:write', res?.ConsumedCapacity?.CapacityUnits ?? 1); diff --git a/src/backend/src/services/PermissionAPIService.js b/src/backend/src/services/PermissionAPIService.js index 94eb6d7f1..22721f898 100644 --- a/src/backend/src/services/PermissionAPIService.js +++ b/src/backend/src/services/PermissionAPIService.js @@ -16,10 +16,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -const { APIError } = require('openai'); +const { APIError } = require('../api/APIError.js'); const configurable_auth = require('../middleware/configurable_auth'); const { Endpoint } = require('../util/expressutil'); -const BaseService = require('./BaseService'); +const { BaseService } = require('./BaseService'); +const { revokeUserAppPermsRoute } = require('../routers/auth/revoke-user-app'); /** * @class PermissionAPIService @@ -44,7 +45,7 @@ class PermissionAPIService extends BaseService { async '__on_install.routes' (_, { app }) { app.use(require('../routers/auth/get-user-app-token')); app.use(require('../routers/auth/grant-user-app')); - app.use(require('../routers/auth/revoke-user-app')); + app.use(revokeUserAppPermsRoute); app.use(require('../routers/auth/grant-dev-app')); app.use(require('../routers/auth/revoke-dev-app')); app.use(require('../routers/auth/grant-user-user')); diff --git a/src/backend/src/services/ai/video/providers/OpenAiVideoGenerationProvider/OpenAIVideoGenerationProvider.ts b/src/backend/src/services/ai/video/providers/OpenAiVideoGenerationProvider/OpenAIVideoGenerationProvider.ts new file mode 100644 index 000000000..94d0fe64e --- /dev/null +++ b/src/backend/src/services/ai/video/providers/OpenAiVideoGenerationProvider/OpenAIVideoGenerationProvider.ts @@ -0,0 +1,246 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import OpenAI from 'openai'; +import APIError from '../../../../../api/APIError.js'; +import { Context } from '../../../../../util/context.js'; +import { MeteringService } from '../../../../MeteringService/MeteringService.js'; +import { IGenerateVideoParams, IVideoModel, IVideoProvider } from '../types.js'; +import { TypedValue } from '../../../../drivers/meta/Runtime.js'; +import { Readable } from 'stream'; +import { OPENAI_VIDEO_MODELS, OPENAI_VIDEO_ALLOWED_SECONDS } from './models.js'; + +const DEFAULT_TEST_VIDEO_URL = 'https://assets.puter.site/txt2vid.mp4'; +const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000; +const POLL_INTERVAL_MS = 5_000; +const DEFAULT_DURATION_SECONDS = 4; + +export class OpenAIVideoGenerationProvider implements IVideoProvider { + #openai: OpenAI; + #meteringService: MeteringService; + + constructor (config: { apiKey: string }, meteringService: MeteringService) { + if ( ! config.apiKey ) { + throw new Error('OpenAI video generation requires an API key'); + } + this.#openai = new OpenAI({ apiKey: config.apiKey }); + this.#meteringService = meteringService; + } + + getDefaultModel (): string { + return OPENAI_VIDEO_MODELS[0].id; + } + + async models (): Promise { + return OPENAI_VIDEO_MODELS; + } + + async generate (params: IGenerateVideoParams): Promise { + const { + prompt, + model: requestedModel, + duration, + seconds, + size, + resolution, + input_reference: inputReference, + test_mode: testMode, + } = params ?? {}; + + if ( typeof prompt !== 'string' || !prompt.trim() ) { + throw APIError.create('field_invalid', null, { + key: 'prompt', + expected: 'a non-empty string', + got: prompt, + }); + } + + const selectedModel = await this.#selectModel(requestedModel); + + if ( ! selectedModel ) { + throw new Error(`Unknown video model: ${requestedModel}`); + } + + if ( testMode ) { + return new TypedValue({ + $: 'string:url:web', + content_type: 'video', + }, DEFAULT_TEST_VIDEO_URL); + } + + const defaultSize = selectedModel.dimensions?.[0] ?? '720x1280'; + const normalizedSize = this.#normalizeSize(size ?? resolution, selectedModel) ?? defaultSize; + const normalizedSeconds = this.#normalizeSeconds(seconds ?? duration) ?? String(DEFAULT_DURATION_SECONDS); + + const sizeTier = this.#determineSizeTier(selectedModel, normalizedSize); + const costPerSecondCents = this.#getCostPerSecond(selectedModel, sizeTier); + + if ( ! costPerSecondCents ) { + throw new Error(`No pricing configured for model ${selectedModel.id} at size ${normalizedSize}`); + } + + const estimatedUnits = this.#parseSeconds(normalizedSeconds) ?? DEFAULT_DURATION_SECONDS; + const actor = Context.get('actor'); + const costInMicroCents = costPerSecondCents * 1_000_000; + const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, costInMicroCents * estimatedUnits); + if ( ! usageAllowed ) { + throw APIError.create('insufficient_funds'); + } + + const createParams: OpenAI.VideoCreateParams = { + prompt, + model: selectedModel.id, + seconds: normalizedSeconds as OpenAI.VideoSeconds, + size: normalizedSize as OpenAI.VideoSize, + }; + + if ( inputReference ) { + createParams.input_reference = inputReference as OpenAI.VideoCreateParams['input_reference']; + } + + const createResponse = await this.#openai.videos.create(createParams); + const finalJob = await this.#pollUntilComplete(createResponse); + + if ( finalJob.status === 'failed' ) { + const errorMessage = finalJob.error?.message ?? 'Video generation failed'; + throw new Error(errorMessage); + } + + const finalResolution = this.#normalizeSize(finalJob.size, selectedModel) ?? normalizedSize; + const finalTier = this.#determineSizeTier(selectedModel, finalResolution); + const finalCostPerSecondCents = this.#getCostPerSecond(selectedModel, finalTier); + + if ( ! finalCostPerSecondCents ) { + throw new Error(`No pricing configured for model ${selectedModel.id} at size ${finalResolution}`); + } + + const finalCostInMicroCents = finalCostPerSecondCents * 1_000_000; + const actualSeconds = this.#parseSeconds(finalJob.seconds) ?? estimatedUnits; + + const downloadResponse = await this.#openai.videos.downloadContent(finalJob.id); + const contentType = downloadResponse.headers.get('content-type') ?? 'video/mp4'; + + let stream: any = downloadResponse.body; + if ( stream && typeof stream.getReader === 'function' ) { + stream = Readable.fromWeb(stream as any); + } + + if ( ! stream ) { + const arrayBuffer = await downloadResponse.arrayBuffer(); + stream = Readable.from(Buffer.from(arrayBuffer)); + } + + const finalUsageKey = this.#getUsageKey(selectedModel, finalTier); + await this.#meteringService.incrementUsage(actor, finalUsageKey, actualSeconds, finalCostInMicroCents * actualSeconds); + + return new TypedValue({ + $: 'stream', + content_type: contentType, + }, stream); + } + + async #selectModel (requestedModel?: string): Promise { + const allModels = await this.models(); + return allModels.find(m => m.id.toLowerCase() === requestedModel?.toLowerCase()); + } + + async #pollUntilComplete (initialJob: OpenAI.Video): Promise { + let job = initialJob; + const start = Date.now(); + + while ( job.status === 'queued' || job.status === 'in_progress' ) { + if ( Date.now() - start > DEFAULT_TIMEOUT_MS ) { + throw new Error('Timed out waiting for Sora video generation to complete'); + } + + await this.#delay(POLL_INTERVAL_MS); + job = await this.#openai.videos.retrieve(job.id); + } + + return job; + } + + async #delay (ms: number): Promise { + return await new Promise(resolve => setTimeout(resolve, ms)); + } + + #normalizeSize (candidate: unknown, model: IVideoModel): string | undefined { + if ( ! candidate ) return undefined; + const normalized = this.#normalizeResolution(candidate); + if ( normalized && model.dimensions?.includes(normalized) ) { + return normalized; + } + return undefined; + } + + #normalizeSeconds (value: unknown): string | undefined { + if ( value === null || value === undefined ) { + return undefined; + } + const parsed = typeof value === 'number' ? String(Math.round(value)) : typeof value === 'string' ? value.trim() : undefined; + if ( parsed && OPENAI_VIDEO_ALLOWED_SECONDS.includes(Number(parsed) as typeof OPENAI_VIDEO_ALLOWED_SECONDS[number]) ) { + return parsed; + } + return undefined; + } + + #determineSizeTier (model: IVideoModel, size: string): string { + if ( model.id === 'sora-2-pro' ) { + if ( size === '1080x1920' || size === '1920x1080' ) return 'xxl'; + if ( size === '1024x1792' || size === '1792x1024' ) return 'xl'; + } + return 'default'; + } + + #getCostPerSecond (model: IVideoModel, tier: string): number | undefined { + const key = tier === 'default' ? 'per-second' : `per-second-${tier}`; + return model.costs?.[key]; + } + + #getUsageKey (model: IVideoModel, tier: string): string { + return `openai:${model.id}:${tier}`; + } + + #normalizeResolution (value: unknown): string | undefined { + if ( ! value ) return undefined; + if ( typeof value === 'string' ) { + const match = value.match(/(\d+)\s*x\s*(\d+)/i); + if ( match ) { + const w = Number.parseInt(match[1], 10); + const h = Number.parseInt(match[2], 10); + if ( Number.isFinite(w) && Number.isFinite(h) ) { + return `${w}x${h}`; + } + } + } + return undefined; + } + + #parseSeconds (value: unknown): number | undefined { + if ( value === null || value === undefined ) return undefined; + if ( typeof value === 'number' && Number.isFinite(value) ) { + return Math.round(value); + } + if ( typeof value === 'string' ) { + const numeric = Number.parseInt(value, 10); + return Number.isFinite(numeric) ? numeric : undefined; + } + return undefined; + } +} diff --git a/src/backend/src/services/ai/video/providers/OpenAiVideoGenerationProvider/models.ts b/src/backend/src/services/ai/video/providers/OpenAiVideoGenerationProvider/models.ts new file mode 100644 index 000000000..061cd341d --- /dev/null +++ b/src/backend/src/services/ai/video/providers/OpenAiVideoGenerationProvider/models.ts @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { IVideoModel } from '../types.js'; + +export const OPENAI_VIDEO_ALLOWED_SECONDS = [4, 8, 12] as const; + +export const OPENAI_VIDEO_MODELS: IVideoModel[] = [ + { + id: 'sora-2', + puterId: 'openai:openai/sora-2', + aliases: ['openai/sora-2'], + name: 'Sora 2', + costs_currency: 'usd-cents', + costs: { + 'per-second': 10, + 'default-duration-per-video': 40, + }, + output_cost_key: 'default-duration-per-video', + durationSeconds: OPENAI_VIDEO_ALLOWED_SECONDS.slice(), + dimensions: ['720x1280', '1280x720'], + defaultUsageKey: 'openai:sora-2:default', + }, + { + id: 'sora-2-pro', + puterId: 'openai:openai/sora-2-pro', + aliases: ['openai/sora-2-pro'], + name: 'Sora 2 Pro', + costs_currency: 'usd-cents', + costs: { + 'per-second': 30, + 'default-duration-per-video': 120, + 'per-second-xl': 50, + 'default-duration-per-video-xl': 200, + 'per-second-xxl': 70, + 'default-duration-per-video-xxl': 280, + }, + output_cost_key: 'default-duration-per-video', + durationSeconds: OPENAI_VIDEO_ALLOWED_SECONDS.slice(), + dimensions: ['720x1280', '1280x720', '1024x1792', '1792x1024', '1080x1920', '1920x1080'], + defaultUsageKey: 'openai:sora-2-pro:default', + }, +]; diff --git a/src/backend/src/services/auth/PermissionService.js b/src/backend/src/services/auth/PermissionService.js index 363f0c678..599b02d71 100644 --- a/src/backend/src/services/auth/PermissionService.js +++ b/src/backend/src/services/auth/PermissionService.js @@ -16,35 +16,25 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -const APIError = require('../../api/APIError'); -const { hardcoded_user_group_permissions } = require('../../data/hardcoded-permissions.js'); -const { ECMAP } = require('../../filesystem/ECMAP'); -const { get_user, get_app } = require('../../helpers'); -const { reading_has_terminal } = require('../../unstructured/permission-scan-lib'); -const { trace } = require('@opentelemetry/api'); -const BaseService = require('../BaseService'); -const { DB_WRITE } = require('../database/consts'); -const { UserActorType, Actor, AppUnderUserActorType } = require('./Actor'); -const { PERM_KEY_PREFIX, MANAGE_PERM_PREFIX } = require('./permissionConts.mjs'); -const { PermissionUtil, PermissionExploder, PermissionImplicator, PermissionRewriter } = require('./permissionUtils.mjs'); -const { spanify } = require('../../util/otelutil'); -const { deleteRedisKeys } = require('../../clients/redis/deleteRedisKeys.js'); -const { setRedisCacheValue } = require('../../clients/redis/cacheUpdate.js'); -const { redisClient } = require('../../clients/redis/redisSingleton'); -const { PermissionScanRedisCacheSpace } = require('./PermissionScanRedisCacheSpace.js'); -const { Context } = require('../../util/context'); +import { trace } from '@opentelemetry/api'; +import { APIError } from '../../api/APIError.js'; +import { setRedisCacheValue } from '../../clients/redis/cacheUpdate.js'; +import { deleteRedisKeys } from '../../clients/redis/deleteRedisKeys.js'; +import { redisClient } from '../../clients/redis/redisSingleton.js'; +import { hardcoded_user_group_permissions } from '../../data/hardcoded-permissions.js'; +import { ECMAP } from '../../filesystem/ECMAP.js'; +import { get_app, get_user } from '../../helpers.js'; +import scanSequence from '../../structured/sequence/scan-permission.mjs'; +import { reading_has_terminal } from '../../unstructured/permission-scan-lib.js'; +import { Context } from '../../util/context.js'; +import { spanify } from '../../util/otelutil.js'; +import { BaseService } from '../BaseService.js'; +import { Actor, UserActorType } from './Actor.js'; +import { MANAGE_PERM_PREFIX, PERM_KEY_PREFIX } from './permissionConts.mjs'; +import { PermissionScanRedisCacheSpace } from './PermissionScanRedisCacheSpace.js'; +import { PermissionExploder, PermissionImplicator, PermissionRewriter, PermissionUtil } from './permissionUtils.mjs'; -/** -* @class PermissionService -* @extends BaseService -* @description -* The PermissionService class manages and enforces permissions within the application. It provides methods to: -* - Check, grant, and revoke permissions for users and applications. -* - Scan for existing permissions. -* - Handle permission implications, rewriting, and explosion to support complex permission hierarchies. -* This service interacts with the database to manage permissions and logs actions for auditing purposes. -*/ -class PermissionService extends BaseService { +export class PermissionService extends BaseService { static CONCERN = 'permissions'; /** * Initializes the PermissionService by setting up internal arrays for permission handling. @@ -66,45 +56,25 @@ class PermissionService extends BaseService { * @throws {Error} If the provided exploder is not an instance of PermissionExploder. */ async _init () { - /** - * @type {import('../../modules/kvstore/KVStoreInterfaceService.js').KVStoreInterface} db - */ - this.kvService = this.services.get('puter-kvstore').as('puter-kvstore'); - this.db = this.services.get('database').get(DB_WRITE, 'permissions'); - this._register_commands(this.services.get('commands')); - this.kvAvgTimes = { count: 0, avg: 0, max: 0 }; - this.dbAvgTimes = { count: 0, avg: 0, max: 0 }; + + this.kvService = this.services.get('puter-kvstore'); + this.db = this.services.get('database'); } async '__on_boot.consolidation' () { const svc_event = this.services.get('event'); // Event to allow extensions to add permissions - { - const event = {}; - event.grant_to_everyone = permission => { - /* eslint-disable */ - hardcoded_user_group_permissions - .system - [this.global_config.default_temp_group] - [permission] - = {}; - hardcoded_user_group_permissions - .system - [this.global_config.default_user_group] - [permission] - = {}; - /* eslint-enable */ - }; - event.grant_to_users = permission => { - /* eslint-disable */ - hardcoded_user_group_permissions - [this.global_config.default_user_group] - [permission] - = {}; - /* eslint-enable */ - }; - svc_event.emit('create.permissions', event); - } + + const event = {}; + event.grant_to_everyone = permission => { + hardcoded_user_group_permissions.system[this.global_config.default_temp_group][permission] = {}; + hardcoded_user_group_permissions.system[this.global_config.default_user_group][permission] = {}; + }; + event.grant_to_users = permission => { + hardcoded_user_group_permissions[this.global_config.default_user_group][permission] = {}; + }; + svc_event.emit('create.permissions', event); + } /** @@ -221,13 +191,12 @@ class PermissionService extends BaseService { // cylog(l, 'ACT & PERM:', actor.uid, permission_options); const start_ts = Date.now(); - await require('../../structured/sequence/scan-permission.mjs').default - .call(this, { - actor, - permission_options, - reading, - state, - }); + await scanSequence.call(this, { + actor, + permission_options, + reading, + state, + }); const end_ts = Date.now(); // TODO: command to enable these logs @@ -1272,64 +1241,4 @@ class PermissionService extends BaseService { this._permission_exploders.push(exploder); } - - _register_commands (commands) { - commands.registerCommands('perms', [ - { - id: 'grant-user-app', - handler: async (args, _log) => { - const [username, app_uid, permission, extra] = args; - - // actor from username - const actor = new Actor({ - type: new UserActorType({ - user: await get_user({ username }), - }), - }); - - await this.grant_user_app_permission(actor, app_uid, permission, extra); - }, - }, - { - id: 'scan', - handler: async (args, ctx) => { - const [username, permission] = args; - - // actor from username - const actor = new Actor({ - type: new UserActorType({ - user: await get_user({ username }), - }), - }); - - let reading = await this.scan(actor, permission); - // reading = PermissionUtil.reading_to_options(reading); - ctx.log(JSON.stringify(reading, undefined, ' ')); - }, - }, - { - id: 'scan-app', - handler: async (args, ctx) => { - const [username, app_name, permission] = args; - const app = await get_app({ name: app_name }); - - // actor from username - const actor = new Actor({ - type: new AppUnderUserActorType({ - app, - user: await get_user({ username }), - }), - }); - - const reading = await this.scan(actor, permission); - // reading = PermissionUtil.reading_to_options(reading); - ctx.log(JSON.stringify(reading, undefined, ' ')); - }, - }, - ]); - } -} - -module.exports = { - PermissionService, -}; +} \ No newline at end of file diff --git a/src/puter-js/src/modules/KV.js b/src/puter-js/src/modules/KV.js index 3b58dd201..00b2dd2b5 100644 --- a/src/puter-js/src/modules/KV.js +++ b/src/puter-js/src/modules/KV.js @@ -26,6 +26,10 @@ const gui_cache_keys = [ 'taskbar_position', 'has_seen_toolbar_animation', ]; + +const isObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +const isOptConfigShorthand = (value) => isObject(value) && Object.prototype.hasOwnProperty.call(value, 'appUuid'); + class KV { MAX_KEY_SIZE = 1024; MAX_VALUE_SIZE = 399 * 1024; @@ -105,16 +109,53 @@ class KV { } /** - * @typedef {function(key: string, value: any, expireAt?: number): Promise} SetFunction + * @typedef {function(key: string, value: any, expireAt?: number, optConfig?: object): Promise} SetFunction * Resolves to 'true' on success, or rejects with an error on failure. * @param {string} key - Cannot be undefined or null. Cannot be larger than 1KB. * @param {any} value - Cannot be larger than 399KB. * @param {number} [expireAt] - Optional expiration time for the key. Note that clients with a clock that is not in sync with the server may experience issues with this method. + * @param {object} [optConfig] - Optional driver call config, e.g. `{ appUuid: 'uuid' }`. * @memberof KV */ /** @type {SetFunction} */ - set = utils.make_driver_method(['key', 'value', 'expireAt'], 'puter-kvstore', undefined, 'set', { + set = async (...args) => { + if ( args.length === 1 && isObject(args[0]) ) { + return await this.set_(args[0]); + } + + const key = args[0]; + const value = args[1]; + const rest = args.slice(2); + + let expireAt; + let optConfig; + let success; + let error; + + if ( typeof rest[0] === 'number' || rest[0] === null ) { + expireAt = rest.shift(); + } + + if ( rest[0] === undefined ) { + rest.shift(); + } + + if ( isObject(rest[0]) ) { + optConfig = rest.shift(); + } + + if ( typeof rest[0] === 'function' ) { + success = rest.shift(); + } + if ( typeof rest[0] === 'function' ) { + error = rest.shift(); + } + + return await this.set_({ key, value, expireAt, optConfig, success, error }); + }; + + set_ = utils.make_driver_method(['key', 'value', 'expireAt'], 'puter-kvstore', undefined, 'set', { /** * * @param {object} args @@ -145,19 +186,39 @@ class KV { * Resolves to the value if the key exists, or `undefined` if the key does not exist. Rejects with an error on failure. */ async get (...args) { + if ( args.length === 1 && isObject(args[0]) ) { + return await this.get_(args[0]); + } + + const key = args[0]; + + let optConfig; + let success; + let error; + + if ( isObject(args[1]) ) { + optConfig = args[1]; + success = args[2]; + error = args[3]; + } else { + success = args[1]; + error = args[2]; + } + // Condition for gui boot cache if ( - typeof args[0] === 'string' && - gui_cache_keys.includes(args[0]) && + typeof key === 'string' && + gui_cache_keys.includes(key) && + !optConfig && this.gui_cached !== null ) { this.gui_cache_init && this.gui_cache_init.resolve(); const cache = await this.gui_cached.promise; - return cache[args[0]]; + return cache[key]; } // Normal get - return await this.get_(...args); + return await this.get_({ key, optConfig, success, error }); } get_ = utils.make_driver_method(['key'], 'puter-kvstore', undefined, 'get', { @@ -177,13 +238,24 @@ class KV { incr = async (...args) => { let options = {}; - // arguments are required - if ( !args || args.length === 0 ) { - throw ({ message: 'Arguments are required', code: 'arguments_required' }); - } + if ( args.length === 1 && isObject(args[0]) ) { + options = { ...args[0] }; + } else { + // arguments are required + if ( !args || args.length === 0 ) { + throw ({ message: 'Arguments are required', code: 'arguments_required' }); + } - options.key = args[0]; - options.pathAndAmountMap = !args[1] ? { '': 1 } : typeof args[1] === 'number' ? { '': args[1] } : args[1]; + options.key = args[0]; + let amountOrMap = args[1]; + let optConfig = args[2]; + if ( isOptConfigShorthand(amountOrMap) && optConfig === undefined ) { + optConfig = amountOrMap; + amountOrMap = undefined; + } + options.pathAndAmountMap = !amountOrMap ? { '': 1 } : typeof amountOrMap === 'number' ? { '': amountOrMap } : amountOrMap; + options.optConfig = optConfig; + } // key size cannot be larger than MAX_KEY_SIZE if ( options.key.length > this.MAX_KEY_SIZE ) { @@ -196,13 +268,24 @@ class KV { decr = async (...args) => { let options = {}; - // arguments are required - if ( !args || args.length === 0 ) { - throw ({ message: 'Arguments are required', code: 'arguments_required' }); - } + if ( args.length === 1 && isObject(args[0]) ) { + options = { ...args[0] }; + } else { + // arguments are required + if ( !args || args.length === 0 ) { + throw ({ message: 'Arguments are required', code: 'arguments_required' }); + } - options.key = args[0]; - options.pathAndAmountMap = !args[1] ? { '': 1 } : typeof args[1] === 'number' ? { '': args[1] } : args[1]; + options.key = args[0]; + let amountOrMap = args[1]; + let optConfig = args[2]; + if ( isOptConfigShorthand(amountOrMap) && optConfig === undefined ) { + optConfig = amountOrMap; + amountOrMap = undefined; + } + options.pathAndAmountMap = !amountOrMap ? { '': 1 } : typeof amountOrMap === 'number' ? { '': amountOrMap } : amountOrMap; + options.optConfig = optConfig; + } // key size cannot be larger than MAX_KEY_SIZE if ( options.key.length > this.MAX_KEY_SIZE ) { @@ -215,15 +298,27 @@ class KV { add = async (...args) => { let options = {}; - // arguments are required - if ( !args || args.length === 0 ) { - throw ({ message: 'Arguments are required', code: 'arguments_required' }); - } + if ( args.length === 1 && isObject(args[0]) ) { + options = { ...args[0] }; + } else { + // arguments are required + if ( !args || args.length === 0 ) { + throw ({ message: 'Arguments are required', code: 'arguments_required' }); + } - options.key = args[0]; - const provided = args[1]; - const isPathMap = provided && typeof provided === 'object' && !Array.isArray(provided); - options.pathAndValueMap = provided === undefined ? { '': 1 } : isPathMap ? provided : { '': provided }; + options.key = args[0]; + + let provided = args[1]; + let optConfig = args[2]; + if ( isOptConfigShorthand(provided) && optConfig === undefined ) { + optConfig = provided; + provided = undefined; + } + + const isPathMap = provided && typeof provided === 'object' && !Array.isArray(provided); + options.pathAndValueMap = provided === undefined ? { '': 1 } : isPathMap ? provided : { '': provided }; + options.optConfig = optConfig; + } // key size cannot be larger than MAX_KEY_SIZE if ( options.key.length > this.MAX_KEY_SIZE ) { @@ -240,6 +335,11 @@ class KV { const key = args[0]; const paths = args.slice(1); + let optConfig; + + if ( isObject(paths[paths.length - 1]) ) { + optConfig = paths.pop(); + } if ( Array.isArray(paths[0]) && paths.length === 1 ) { throw ({ message: 'Paths must be provided as separate arguments', code: 'paths_invalid' }); @@ -262,10 +362,43 @@ class KV { } return utils.make_driver_method(['key', 'paths'], 'puter-kvstore', undefined, 'remove') - .call(this, { key, paths }); + .call(this, { key, paths, optConfig }); }; - update = utils.make_driver_method(['key', 'pathAndValueMap', 'ttl'], 'puter-kvstore', undefined, 'update', { + update = async (...args) => { + if ( args.length === 1 && isObject(args[0]) ) { + return await this.update_(args[0]); + } + + const key = args[0]; + const pathAndValueMap = args[1]; + + let ttl; + let optConfig; + let success; + let error; + + const rest = args.slice(2); + if ( typeof rest[0] === 'number' || rest[0] === null ) { + ttl = rest.shift(); + } + if ( rest[0] === undefined ) { + rest.shift(); + } + if ( isObject(rest[0]) ) { + optConfig = rest.shift(); + } + if ( typeof rest[0] === 'function' ) { + success = rest.shift(); + } + if ( typeof rest[0] === 'function' ) { + error = rest.shift(); + } + + return await this.update_({ key, pathAndValueMap, ttl, optConfig, success, error }); + }; + + update_ = utils.make_driver_method(['key', 'pathAndValueMap', 'ttl'], 'puter-kvstore', undefined, 'update', { preprocess: (args) => { if ( args.key === undefined || args.key === null ) { throw { message: 'Key cannot be undefined', code: 'key_undefined' }; @@ -298,10 +431,11 @@ class KV { * @memberof [KV] * @returns */ - expire = async (key, ttl) => { + expire = async (key, ttl, optConfig) => { let options = {}; options.key = key; options.ttl = ttl; + options.optConfig = optConfig; // key size cannot be larger than MAX_KEY_SIZE if ( options.key.length > this.MAX_KEY_SIZE ) { @@ -320,10 +454,11 @@ class KV { * @memberof [KV] * @returns */ - expireAt = async (key, timestamp) => { + expireAt = async (key, timestamp, optConfig) => { let options = {}; options.key = key; options.timestamp = timestamp; + options.optConfig = optConfig; // key size cannot be larger than MAX_KEY_SIZE if ( options.key.length > this.MAX_KEY_SIZE ) { throw ({ message: `Key size cannot be larger than ${this.MAX_KEY_SIZE}`, code: 'key_too_large' }); @@ -334,7 +469,29 @@ class KV { // resolves to 'true' on success, or rejects with an error on failure // will still resolve to 'true' if the key does not exist - del = utils.make_driver_method(['key'], 'puter-kvstore', undefined, 'del', { + del = async (...args) => { + if ( args.length === 1 && isObject(args[0]) ) { + return await this.del_(args[0]); + } + + const key = args[0]; + let optConfig; + let success; + let error; + + if ( isObject(args[1]) ) { + optConfig = args[1]; + success = args[2]; + error = args[3]; + } else { + success = args[1]; + error = args[2]; + } + + return await this.del_({ key, optConfig, success, error }); + }; + + del_ = utils.make_driver_method(['key'], 'puter-kvstore', undefined, 'del', { preprocess: (args) => { // key size cannot be larger than this.MAX_KEY_SIZE if ( args.key.length > this.MAX_KEY_SIZE ) { @@ -358,6 +515,11 @@ class KV { pattern = input.pattern; } returnValues = !!input.returnValues; + if ( isObject(input.optConfig) ) { + options.optConfig = input.optConfig; + } else if ( isOptConfigShorthand(input) ) { + options.optConfig = input; + } if ( input.limit !== undefined ) { options.limit = input.limit; } @@ -365,16 +527,36 @@ class KV { options.cursor = input.cursor; } } else { + if ( isObject(args[1]) ) { + options.optConfig = args[1]; + } + + if ( isObject(args[2]) ) { + options.optConfig = args[2]; + } + // list(true) or list(pattern, true) will return the key-value pairs if ( (args && args.length === 1 && args[0] === true) || (args && args.length === 2 && args[1] === true) ) { returnValues = true; } + if ( args && args.length === 3 && args[1] === true ) { + returnValues = true; + } + // list(pattern) // list(pattern, true) + // list(pattern, optConfig) + // list(pattern, true, optConfig) if ( (args && args.length === 1 && typeof args[0] === 'string') || (args && args.length === 2 && typeof args[0] === 'string' && args[1] === true) ) { pattern = args[0]; } + if ( args && args.length === 2 && typeof args[0] === 'string' && isObject(args[1]) ) { + pattern = args[0]; + } + if ( args && args.length === 3 && typeof args[0] === 'string' && args[1] === true ) { + pattern = args[0]; + } } if ( ! returnValues ) { @@ -391,7 +573,37 @@ class KV { // resolve to 'true' on success, or rejects with an error on failure // will still resolve to 'true' if there are no keys - flush = utils.make_driver_method([], 'puter-kvstore', undefined, 'flush'); + flush = async (...args) => { + if ( args.length === 1 && isObject(args[0]) ) { + const input = args[0]; + if ( + Object.prototype.hasOwnProperty.call(input, 'optConfig') || + Object.prototype.hasOwnProperty.call(input, 'success') || + Object.prototype.hasOwnProperty.call(input, 'error') + ) { + return await this.flush_(input); + } + + return await this.flush_({ optConfig: input }); + } + + let optConfig; + let success; + let error; + + if ( isObject(args[0]) ) { + optConfig = args[0]; + success = args[1]; + error = args[2]; + } else { + success = args[0]; + error = args[1]; + } + + return await this.flush_({ optConfig, success, error }); + }; + + flush_ = utils.make_driver_method([], 'puter-kvstore', undefined, 'flush'); // clear is an alias for flush clear = this.flush; diff --git a/src/puter-js/test/kv.test.js b/src/puter-js/test/kv.test.js index a4f2fd2f8..2fd935e4e 100644 --- a/src/puter-js/test/kv.test.js +++ b/src/puter-js/test/kv.test.js @@ -319,6 +319,78 @@ window.kvTests = [ } } }, + { + name: "testOptConfigNamespaceIsolation", + description: "Test that optConfig.appUuid isolates KV namespaces", + test: async function() { + try { + const suffix = puter.randName(); + const key = 'optConfigKey-' + suffix; + const overrideA = { appUuid: 'opt-app-a-' + suffix }; + const overrideB = { appUuid: 'opt-app-b-' + suffix }; + + await puter.kv.set(key, 'default-value'); + await puter.kv.set(key, 'override-a-value', overrideA); + await puter.kv.set(key, 'override-b-value', overrideB); + + const defaultValue = await puter.kv.get(key); + const overrideAValue = await puter.kv.get(key, overrideA); + const overrideBValue = await puter.kv.get(key, overrideB); + + assert(defaultValue === 'default-value', "Default namespace value mismatch"); + assert(overrideAValue === 'override-a-value', "Override A value mismatch"); + assert(overrideBValue === 'override-b-value', "Override B value mismatch"); + + const listA = await puter.kv.list(key + '*', overrideA); + assert(Array.isArray(listA), "Expected list result to be an array"); + assert(listA.includes(key), "Override A list should include the key"); + + await puter.kv.del(key, overrideA); + const afterDeleteOverride = await puter.kv.get(key, overrideA); + const afterDeleteDefault = await puter.kv.get(key); + + assert(afterDeleteOverride === null, "Override A key should be deleted"); + assert(afterDeleteDefault === 'default-value', "Default namespace should remain untouched"); + pass("testOptConfigNamespaceIsolation passed"); + } catch (error) { + fail("testOptConfigNamespaceIsolation failed:", error); + } + } + }, + { + name: "testOptConfigShorthandAndScopedFlush", + description: "Test optConfig shorthand calls and namespace-scoped flush", + test: async function() { + try { + const suffix = puter.randName(); + const overrideA = { appUuid: 'opt-shorthand-a-' + suffix }; + const overrideB = { appUuid: 'opt-shorthand-b-' + suffix }; + const counterKey = 'optCounter-' + suffix; + const updateKey = 'optUpdate-' + suffix; + const flushKeyA = 'optFlushA-' + suffix; + const flushKeyB = 'optFlushB-' + suffix; + + const incrResult = await puter.kv.incr(counterKey, overrideA); + assert(incrResult === 1, "Expected shorthand incr to initialize counter to 1"); + assert(await puter.kv.get(counterKey) === null, "Default namespace counter should remain unset"); + assert(await puter.kv.get(counterKey, overrideA) === 1, "Override namespace counter mismatch"); + + const updateResult = await puter.kv.update(updateKey, { 'profile.name': 'Ada' }, overrideA); + assert(updateResult?.profile?.name === 'Ada', "Expected update in override namespace to succeed"); + assert(await puter.kv.get(updateKey) === null, "Default namespace update key should remain unset"); + + await puter.kv.set(flushKeyA, 'A', overrideA); + await puter.kv.set(flushKeyB, 'B', overrideB); + await puter.kv.flush(overrideA); + + assert(await puter.kv.get(flushKeyA, overrideA) === null, "Scoped flush should clear override A keys"); + assert(await puter.kv.get(flushKeyB, overrideB) === 'B', "Scoped flush should not clear override B keys"); + pass("testOptConfigShorthandAndScopedFlush passed"); + } catch (error) { + fail("testOptConfigShorthandAndScopedFlush failed:", error); + } + } + }, { name: "testIncr", description: "Test incrementing a key and verify it returns 1", diff --git a/src/puter-js/types/modules/kv.d.ts b/src/puter-js/types/modules/kv.d.ts index f8fef11fa..d8a7f895b 100644 --- a/src/puter-js/types/modules/kv.d.ts +++ b/src/puter-js/types/modules/kv.d.ts @@ -24,6 +24,7 @@ export interface KVListOptions { returnValues?: boolean; limit?: number; cursor?: string; + optConfig?: KVOptConfig; } export type KVListPaginationOptions = @@ -35,27 +36,39 @@ export interface KVListPage { cursor?: string; } +export interface KVOptConfig { + appUuid?: string; +} + export class KV { readonly MAX_KEY_SIZE: number; readonly MAX_VALUE_SIZE: number; - set(key: string, value: T, expireAt?: number): Promise; - get(key: string): Promise; - del (key: string): Promise; - incr (key: string, amount?: number | KVIncrementPath): Promise; - decr (key: string, amount?: number | KVIncrementPath): Promise; - add (key: string, value?: KVValue | KVAddPath): Promise; - remove (key: string, ...paths: string[]): Promise; - update (key: string, pathAndValueMap: KVUpdatePath, ttlSeconds?: number): Promise; - expire (key: string, ttlSeconds: number): Promise; - expireAt (key: string, timestampSeconds: number): Promise; + set(key: string, value: T, optConfig: KVOptConfig): Promise; + set(key: string, value: T, expireAt?: number, optConfig?: KVOptConfig): Promise; + get(key: string, optConfig?: KVOptConfig): Promise; + del (key: string, optConfig?: KVOptConfig): Promise; + incr (key: string, optConfig: KVOptConfig): Promise; + incr (key: string, amount?: number | KVIncrementPath, optConfig?: KVOptConfig): Promise; + decr (key: string, optConfig: KVOptConfig): Promise; + decr (key: string, amount?: number | KVIncrementPath, optConfig?: KVOptConfig): Promise; + add (key: string, optConfig: KVOptConfig): Promise; + add (key: string, value?: KVValue | KVAddPath, optConfig?: KVOptConfig): Promise; + remove (key: string, ...paths: Array): Promise; + update (key: string, pathAndValueMap: KVUpdatePath, optConfig: KVOptConfig): Promise; + update (key: string, pathAndValueMap: KVUpdatePath, ttlSeconds?: number, optConfig?: KVOptConfig): Promise; + expire (key: string, ttlSeconds: number, optConfig?: KVOptConfig): Promise; + expireAt (key: string, timestampSeconds: number, optConfig?: KVOptConfig): Promise; list (pattern?: string, returnValues?: false): Promise; list(pattern: string, returnValues: true): Promise[]>; list(returnValues: true): Promise[]>; + list (pattern: string, returnValues: boolean, optConfig: KVOptConfig): Promise[]>; + list (pattern: string, optConfig: KVOptConfig): Promise; + list(returnValues: true, optConfig: KVOptConfig): Promise[]>; list (options: KVListOptions & KVListPaginationOptions & { returnValues?: false }): Promise>; list(options: KVListOptions & KVListPaginationOptions & { returnValues: true }): Promise>>; list (options: KVListOptions & { returnValues?: false }): Promise; list(options: KVListOptions & { returnValues: true }): Promise[]>; - flush (): Promise; - clear (): Promise; + flush (optConfig?: KVOptConfig): Promise; + clear (optConfig?: KVOptConfig): Promise; } diff --git a/tests/puterJsApiTests/kv.test.ts b/tests/puterJsApiTests/kv.test.ts index cec95d083..701f84931 100644 --- a/tests/puterJsApiTests/kv.test.ts +++ b/tests/puterJsApiTests/kv.test.ts @@ -218,6 +218,54 @@ describe('Puter KV Module', () => { expect(Array.isArray(secondPageObj.items)).toBe(true); expect(secondPageObj.items.length).toBeLessThanOrEqual(1); }); + it('should isolate namespaces when using optConfig.appUuid', async () => { + const suffix = Date.now().toString(36); + const key = `${TEST_KEY}-opt-override-${suffix}`; + const overrideA = { appUuid: `opt-a-${suffix}` }; + const overrideB = { appUuid: `opt-b-${suffix}` }; + + await puter.kv.set(key, 'default-value'); + await puter.kv.set(key, 'override-a-value', overrideA); + await puter.kv.set(key, 'override-b-value', overrideB); + + expect(await puter.kv.get(key)).toBe('default-value'); + expect(await puter.kv.get(key, overrideA)).toBe('override-a-value'); + expect(await puter.kv.get(key, overrideB)).toBe('override-b-value'); + + const listA = await puter.kv.list(`${key}*`, overrideA); + expect(Array.isArray(listA)).toBe(true); + expect((listA as string[])).toContain(key); + + await puter.kv.del(key, overrideA); + expect(await puter.kv.get(key, overrideA)).toBeNull(); + expect(await puter.kv.get(key)).toBe('default-value'); + }); + it('should support optConfig shorthand and scoped flush', async () => { + const suffix = Date.now().toString(36); + const overrideA = { appUuid: `opt-shorthand-a-${suffix}` }; + const overrideB = { appUuid: `opt-shorthand-b-${suffix}` }; + const counterKey = `${TEST_KEY}-opt-counter-${suffix}`; + const updateKey = `${TEST_KEY}-opt-update-${suffix}`; + const flushKeyA = `${TEST_KEY}-opt-flush-a-${suffix}`; + const flushKeyB = `${TEST_KEY}-opt-flush-b-${suffix}`; + + const incrRes = await puter.kv.incr(counterKey, overrideA); + expect(incrRes).toBe(1); + expect(await puter.kv.get(counterKey)).toBeNull(); + expect(await puter.kv.get(counterKey, overrideA)).toBe(1); + + const updateRes = await puter.kv.update(updateKey, { 'profile.name': 'Ada' }, overrideA); + expect(updateRes).toEqual({ profile: { name: 'Ada' } }); + expect(await puter.kv.get(updateKey)).toBeNull(); + expect(await puter.kv.get(updateKey, overrideA)).toEqual({ profile: { name: 'Ada' } }); + + await puter.kv.set(flushKeyA, 'A', overrideA); + await puter.kv.set(flushKeyB, 'B', overrideB); + await puter.kv.flush(overrideA); + + expect(await puter.kv.get(flushKeyA, overrideA)).toBeNull(); + expect(await puter.kv.get(flushKeyB, overrideB)).toBe('B'); + }); // delete ops should go last it('should flush all keys', async () => { const flushRes = await puter.kv.flush();