diff --git a/src/backend/clients/database/SqliteDatabaseClient.test.ts b/src/backend/clients/database/SqliteDatabaseClient.test.ts index f5e0523ff..c74bffa7d 100644 --- a/src/backend/clients/database/SqliteDatabaseClient.test.ts +++ b/src/backend/clients/database/SqliteDatabaseClient.test.ts @@ -27,7 +27,7 @@ import { DatabaseClientFactory } from './index.js'; import { SqliteDatabaseClient } from './SqliteDatabaseClient.js'; /** Highest schema version the migration table can reach. */ -const CURRENT_SCHEMA_VERSION = 71; +const CURRENT_SCHEMA_VERSION = 72; /** * These suites migrate real files on disk. Idle they finish in well under a diff --git a/src/backend/clients/database/SqliteDatabaseClient.ts b/src/backend/clients/database/SqliteDatabaseClient.ts index 067638b02..0c28b2d78 100644 --- a/src/backend/clients/database/SqliteDatabaseClient.ts +++ b/src/backend/clients/database/SqliteDatabaseClient.ts @@ -105,6 +105,7 @@ const AVAILABLE_MIGRATIONS: [number, string[]][] = [ [68, ['0073_notification-created-at.sql']], [69, ['0074_add_user_home.sql']], [70, ['0075_event-subscriptions.sql']], + [71, ['0076_event-handlers.sql']], ]; export class SqliteDatabaseClient extends AbstractDatabaseClient { diff --git a/src/backend/clients/database/migrations/mysql/mysql_mig_31.sql b/src/backend/clients/database/migrations/mysql/mysql_mig_31.sql new file mode 100644 index 000000000..196805eb2 --- /dev/null +++ b/src/backend/clients/database/migrations/mysql/mysql_mig_31.sql @@ -0,0 +1,37 @@ +-- Copyright (C) 2024-present Puter Technologies Inc. +-- +-- This file is part of Puter. +-- +-- Puter is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as published +-- by the Free Software Foundation, either version 3 of the License, or +-- (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU Affero General Public License for more details. +-- +-- You should have received a copy of the GNU Affero General Public License +-- along with this program. If not, see . + +-- Named handlers an app deploys once. See sqlite/0076_event-handlers.sql for +-- the column rationale. +-- +-- Idempotent: `CREATE TABLE IF NOT EXISTS` with the indexes declared inline, +-- as mig_28. There is no per-file applied-state tracking, so a replay has to +-- be a no-op. + +CREATE TABLE IF NOT EXISTS `event_handlers` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + -- Matches `apps`.`uid` exactly, charset included, so an equality against + -- one never falls back to a conversion. + `app_uid` char(40) CHARACTER SET latin1 COLLATE latin1_swedish_ci NOT NULL, + `name` varchar(128) NOT NULL, + `source` mediumtext NOT NULL, + `source_hash` char(64) NOT NULL, + `created_at` bigint NOT NULL, + `updated_at` bigint NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `idx_event_handlers_app_name` (`app_uid`, `name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/src/backend/clients/database/migrations/postgres/postgres_mig_20.sql b/src/backend/clients/database/migrations/postgres/postgres_mig_20.sql new file mode 100644 index 000000000..f3a1052d0 --- /dev/null +++ b/src/backend/clients/database/migrations/postgres/postgres_mig_20.sql @@ -0,0 +1,34 @@ +-- Copyright (C) 2024-present Puter Technologies Inc. +-- +-- This file is part of Puter. +-- +-- Puter is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as published +-- by the Free Software Foundation, either version 3 of the License, or +-- (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU Affero General Public License for more details. +-- +-- You should have received a copy of the GNU Affero General Public License +-- along with this program. If not, see . + +-- Named handlers an app deploys once. See sqlite/0076_event-handlers.sql for +-- the column rationale. +-- +-- Idempotent via IF NOT EXISTS. + +CREATE TABLE IF NOT EXISTS event_handlers ( + id BIGSERIAL PRIMARY KEY, + app_uid VARCHAR(40) NOT NULL, + name VARCHAR(128) NOT NULL, + source TEXT NOT NULL, + source_hash VARCHAR(64) NOT NULL, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_event_handlers_app_name + ON event_handlers (app_uid, name); diff --git a/src/backend/clients/database/migrations/sqlite/0076_event-handlers.sql b/src/backend/clients/database/migrations/sqlite/0076_event-handlers.sql new file mode 100644 index 000000000..2139c42ab --- /dev/null +++ b/src/backend/clients/database/migrations/sqlite/0076_event-handlers.sql @@ -0,0 +1,50 @@ +-- Copyright (C) 2024-present Puter Technologies Inc. +-- +-- This file is part of Puter. +-- +-- Puter is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as published +-- by the Free Software Foundation, either version 3 of the License, or +-- (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU Affero General Public License for more details. +-- +-- You should have received a copy of the GNU Affero General Public License +-- along with this program. If not, see . + +-- Named handlers an app deploys once and its users' subscriptions bind to by +-- name. A handler is an addressable object with its own lifecycle: nothing +-- triggers by name, and a row here runs only when a subscription bound to it +-- has a delivery. +-- +-- - `app_uid` : the namespace. Names are unique per app, and there is no +-- foreign key for the same reason `event_subscriptions` +-- has none — a row has to stay manageable after the app +-- row moves. +-- - `name` : the identity a subscription binds to, stable across +-- source changes. +-- - `source` : the serialized function. Read only by the delivery path +-- and never returned by `list`. +-- - `source_hash` : change detector and idempotency key. A publish carrying +-- the same hash is a no-op, and a subscription sending an +-- inline hash binds only when it matches this one. +-- +-- `created_at` / `updated_at` are unix seconds, matching `event_subscriptions`. + +CREATE TABLE IF NOT EXISTS `event_handlers` ( + "id" INTEGER PRIMARY KEY AUTOINCREMENT, + "app_uid" TEXT NOT NULL, + "name" TEXT NOT NULL, + "source" TEXT NOT NULL, + "source_hash" TEXT NOT NULL, + "created_at" INTEGER NOT NULL, -- unix seconds + "updated_at" INTEGER NOT NULL +); + +-- The name is the identity, and it is unique inside one app. This index is +-- also the lookup a subscribe binding check runs on. +CREATE UNIQUE INDEX IF NOT EXISTS `idx_event_handlers_app_name` + ON `event_handlers` (`app_uid`, `name`); diff --git a/src/backend/controllers/events/EventsController.ts b/src/backend/controllers/events/EventsController.ts index 7c80ce7ac..5ff57098f 100644 --- a/src/backend/controllers/events/EventsController.ts +++ b/src/backend/controllers/events/EventsController.ts @@ -24,7 +24,7 @@ import { HttpError } from '../../core/http/HttpError.js'; import { DURABLE_LIST_LIMIT_CAP } from '../../stores/events/DurableSubscriptionStore.js'; import { normalizeLimit } from '../../util/pagination.js'; import { PuterController } from '../types.js'; -import { EVENTS_LIST_LIMIT } from './limits.js'; +import { EVENTS_HANDLER_LIST_LIMIT, EVENTS_LIST_LIMIT } from './limits.js'; /** * The durable half of the events surface. Session subscriptions arrive over the @@ -99,6 +99,72 @@ export class EventsController extends PuterController { res.json({}); } + // -- Handlers ---------------------------------------------------- + // + // Deploying an app's code, so the gate is the same as the verbs above plus + // the ownership check the service makes: the app token's own app, or an app + // a user session names and that user owns. + + /** POST /events/handlers/publish — create or update one named handler. */ + @Post('/handlers/publish', { + subdomain: 'api', + requireAuth: true, + allowAccessToken: true, + }) + async publishHandler(req: Request, res: Response): Promise { + const actor = this.#requireActor(req); + res.json( + await this.services.events.publishHandler(actor, this.#body(req)), + ); + } + + /** POST /events/handlers/publishAll — a build step's whole set, in order. */ + @Post('/handlers/publishAll', { + subdomain: 'api', + requireAuth: true, + allowAccessToken: true, + }) + async publishHandlers(req: Request, res: Response): Promise { + const actor = this.#requireActor(req); + res.json({ + handlers: await this.services.events.publishHandlers( + actor, + this.#body(req), + ), + }); + } + + /** GET /events/handlers/list — names and hashes, never source. */ + @Get('/handlers/list', { + subdomain: 'api', + requireAuth: true, + allowAccessToken: true, + rateLimit: EVENTS_HANDLER_LIST_LIMIT, + }) + async listHandlers(req: Request, res: Response): Promise { + const actor = this.#requireActor(req); + const query = (req.query ?? {}) as Record; + res.json({ + handlers: await this.services.events.listHandlers(actor, { + appUid: + typeof query.appUid === 'string' ? query.appUid : undefined, + }), + }); + } + + /** POST /events/handlers/remove — delete, and suspend what was bound. */ + @Post('/handlers/remove', { + subdomain: 'api', + requireAuth: true, + allowAccessToken: true, + }) + async removeHandler(req: Request, res: Response): Promise { + const actor = this.#requireActor(req); + res.json( + await this.services.events.removeHandler(actor, this.#body(req)), + ); + } + // -- Internals --------------------------------------------------- #requireActor(req: Request): Actor { diff --git a/src/backend/controllers/events/limits.ts b/src/backend/controllers/events/limits.ts index cba4f0d1a..6ec9ba37e 100644 --- a/src/backend/controllers/events/limits.ts +++ b/src/backend/controllers/events/limits.ts @@ -93,6 +93,45 @@ export const EVENTS_LIST_LIMIT = userWindow('events:list', 120); */ export const EVENTS_ACK_LIMIT = userWindow('events:ack', 600); +// -- Handler surface ------------------------------------------------- + +/** + * Named handlers one app may have published. + * + * A handler is a row read on the delivery path, so the cap is on how many + * distinct pieces of code an app asks the system to keep addressable — not on + * how often it changes them. An app past this is describing events by name + * where a `match` filter belongs. + */ +export const EVENTS_HANDLERS_PER_APP = 100; + +/** Longest a serialized handler may be. */ +export const EVENTS_HANDLER_SOURCE_MAX_BYTES = 64 * 1024; + +/** + * Handlers one `publishAll` may carry. A build step publishes its whole set in + * one call, and the set is capped by what an app may hold anyway. + */ +export const EVENTS_HANDLER_PUBLISH_BATCH = 50; + +/** + * Handler publishes and removals per minute, per user. + * + * A build step publishes its whole set in one call and a developer iterating + * publishes a handful; level with the subscribe budget, which is the closest + * analogue — a write a client makes deliberately, never in a loop. + */ +export const EVENTS_HANDLER_PUBLISH_LIMIT = userWindow( + 'events:handlers:publish', + 60, +); + +/** Handler listings per minute, per user. Reads an index, so budgeted higher. */ +export const EVENTS_HANDLER_LIST_LIMIT = userWindow( + 'events:handlers:list', + 120, +); + // -- Dispatch fan-out ------------------------------------------------ /** @@ -144,6 +183,29 @@ export const EVENTS_PENDING_DELIVERIES_PER_SUBSCRIPTION = 10_000; */ export const EVENTS_REGION_PENDING_CEILING = 1_000_000; +// -- Suspended backlog ----------------------------------------------- +// +// A suspended subscription stops metering but keeps what it is owed, and that +// pair is a free memory hold: removing one widely-subscribed handler would +// otherwise turn every dependent into a full backlog nobody pays for. So a +// suspension trims to a much smaller cap and stamps an expiry, and the pending +// sweeper drops what is left over with a gap marker in its place. + +/** Deliveries a suspended subscription may keep, whatever the reason. */ +export const EVENTS_SUSPENDED_PENDING_CAP = 100; + +/** + * How long a backlog held for a handler that may come back is kept. A bad + * deploy is recoverable within a day; past that the events are stale anyway. + */ +export const EVENTS_SUSPENDED_BACKLOG_TTL_MS = 24 * 60 * 60 * 1000; + +/** + * How long a backlog held for an account out of credit is kept. The resume + * condition is a top-up, which is usually minutes. + */ +export const EVENTS_NO_CREDIT_BACKLOG_TTL_MS = 60 * 60 * 1000; + // -- Coalescing ------------------------------------------------------ /** diff --git a/src/backend/services/events/EventsService.ts b/src/backend/services/events/EventsService.ts index 605be8ea5..308941b99 100644 --- a/src/backend/services/events/EventsService.ts +++ b/src/backend/services/events/EventsService.ts @@ -23,6 +23,8 @@ import { EVENTS_ACK_LIMIT, EVENTS_BROADCAST_DELIVERY_LIMIT, EVENTS_COALESCE_WINDOW_MS, + EVENTS_HANDLER_PUBLISH_BATCH, + EVENTS_HANDLER_PUBLISH_LIMIT, EVENTS_MATCHED_SUBSCRIPTIONS_PER_EVENT, EVENTS_SUBSCRIBE_LIMIT, SUSPENDED_ROW_TTL_DAYS, @@ -30,7 +32,20 @@ import { import type { Actor } from '../../core/actor.js'; import { HttpError } from '../../core/http/HttpError.js'; import { checkRateLimit } from '../../core/http/middleware/rateLimit.js'; -import type { ReanchorInput } from '../../stores/events/DurableSubscriptionStore.js'; +import type { + ReanchorInput, + SuspendedReason, +} from '../../stores/events/DurableSubscriptionStore.js'; +import { + HANDLER_SETTLE_BATCH, + isSuspendedReason, +} from '../../stores/events/DurableSubscriptionStore.js'; +import { + HANDLER_NAME_MAX_LENGTH, + hashContent, + type EventHandlerSummary, + type PublishOutcome, +} from '../../stores/events/EventHandlerStore.js'; import { SESSION_SUBSCRIPTION_TTL_SECONDS, type DispatchSubscription, @@ -113,6 +128,7 @@ import { type ParsedSubject, type SubjectOp, } from './subjects.js'; +import { backlogPolicyFor, isResumable } from './suspension.js'; import { RecordingWorkerInvoker, type WorkerInvocation, @@ -163,10 +179,59 @@ export interface DurableSubscribeRequest extends SubscribeRequest { delivery?: unknown; targets?: unknown; handlerName?: unknown; + /** + * Hash of the source the caller believes `handlerName` is published with. + * Sent when the subscribe passed an inline handler; the row binds only if + * it matches what the app actually published. + */ + handlerHash?: unknown; context?: unknown; expiresAt?: unknown; } +/** Body of `POST /events/handlers/publish`. */ +export interface PublishHandlerRequest { + appUid?: unknown; + name?: unknown; + source?: unknown; + /** The published hash this publish is an update to. */ + ifHash?: unknown; + replace?: unknown; +} + +/** Body of `POST /events/handlers/publishAll`. */ +export interface PublishHandlersRequest { + appUid?: unknown; + handlers?: unknown; +} + +/** Body of `POST /events/handlers/remove`, and the shape `list` is scoped by. */ +export interface HandlerNameRequest { + appUid?: unknown; + name?: unknown; +} + +export interface HandlerScopeRequest { + appUid?: unknown; +} + +/** What one publish reports back. Never carries source. */ +export interface PublishedHandlerView { + name: string; + hash: string; + updatedAt: number; + outcome: PublishOutcome; + /** Suspended subscriptions this publish brought back into service. */ + resumed: number; +} + +/** What a removal did to the name and to whatever was bound to it. */ +export interface RemovedHandlerView { + name: string; + removed: boolean; + suspended: number; +} + export interface DurableListRequest { limit?: number; cursor?: string; @@ -184,14 +249,21 @@ export interface SubscriptionView { } /** - * A durable row as its holder sees it. `context` is deliberately absent: it is - * read on the delivery path and nowhere else, and a listing is the one surface - * an app can call repeatedly. + * A durable row as its holder sees it. `context` **values** are deliberately + * absent: the column holds whatever secret the handler needs, it is read on the + * delivery path and nowhere else, and a listing is the one surface an app can + * call repeatedly. What is safe to return is the shape — which keys are set, + * and a hash that changes when any value does, which is what lets a caller tell + * two subscriptions apart without being handed either one's secrets. */ export interface DurableSubscriptionView extends SubscriptionView { delivery: DeliveryClass; handlerName: string | null; appUid: string | null; + /** Key names of the stored context, or `null` for a row without one. */ + contextKeys: string[] | null; + /** Hash of the stored context, so a change is visible without the values. */ + contextHash: string | null; createdAt: number; expiresAt: number | null; suspendedAt: number | null; @@ -361,6 +433,42 @@ const handlerRequired = (): HttpError => legacyCode: 'events_handler_required', }); +const handlerNotFound = (name: string): HttpError => + new HttpError(404, `No handler named \`${name}\` is published`, { + legacyCode: 'events_handler_not_found', + }); + +/** + * The inline body the caller sent is not what is published. Refused rather than + * bound: the point of sending a hash is to find out, and binding the published + * source anyway would run code the caller never saw. + */ +const handlerHashMismatch = (name: string): HttpError => + new HttpError( + 409, + `The handler published as \`${name}\` is not the source this subscription was written against`, + { legacyCode: 'events_handler_hash_mismatch' }, + ); + +/** + * Handlers belong to an app, so a caller has to be acting for one — an app + * token names its own, and a user session names one in the request. + */ +const handlerAppRequired = (): HttpError => + new HttpError(400, 'Publishing a handler requires an app', { + legacyCode: 'events_handler_app_required', + }); + +/** + * Deploying an app's code is the developer's, and an app token cannot borrow + * its user's ownership of some other app. Same answer for an app that is not + * there: which apps exist is not this surface's to disclose. + */ +const handlerAppForbidden = (): HttpError => + new HttpError(403, 'Only the app owner may publish its handlers', { + legacyCode: 'events_handler_forbidden', + }); + const badRequest = (message: string, code: string): HttpError => new HttpError(400, message, { legacyCode: code }); @@ -388,11 +496,32 @@ const toView = (sub: DispatchSubscription): SubscriptionView => ({ targets: sub.targets ?? SESSION_TARGETS, }); +/** + * The context as a listing may describe it: which keys it sets, and a hash of + * the whole blob. Never the values — the column is where an API key lives. + */ +const projectContext = ( + context: string | null, +): Pick => { + if (context === null) return { contextKeys: null, contextHash: null }; + let keys: string[] = []; + try { + const parsed: unknown = JSON.parse(context); + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) + keys = Object.keys(parsed).sort(); + } catch { + // Stored by this service as JSON, so this cannot normally happen; an + // unreadable blob still reports its hash rather than failing the list. + } + return { contextKeys: keys, contextHash: hashContent(context) }; +}; + const toDurableView = (sub: DurableSubscription): DurableSubscriptionView => ({ ...toView(sub), delivery: sub.delivery, handlerName: sub.handlerName, appUid: sub.appUid, + ...projectContext(sub.context), createdAt: sub.createdAt, expiresAt: sub.expiresAt, suspendedAt: sub.suspendedAt, @@ -465,9 +594,6 @@ const isCrossAppKvRow = ( // -- Durable request parsing ------------------------------------------ -/** Longest a `handlerName` may be, matching the column that holds it. */ -const HANDLER_NAME_MAX_LENGTH = 128; - const parseDelivery = (value: unknown): DeliveryClass => { if (value === undefined || value === null || value === 'broadcast') return 'broadcast'; @@ -475,11 +601,20 @@ const parseDelivery = (value: unknown): DeliveryClass => { throw badRequest(`Unknown delivery class: ${String(value)}`, 'bad_request'); }; +/** + * There is exactly one events worker per app, so a row with no app has no + * worker to invoke — `targets` for one may only ever carry `socket`. Omitted + * targets default there quietly; an explicit ask for `worker` is refused rather + * than silently dropped, since that is the caller telling us it expected + * background delivery to exist. + */ const parseTargets = ( value: unknown, delivery: DeliveryClass, + appUid: string | null, ): SubscriptionTarget[] => { - if (value === undefined || value === null) return DEFAULT_DURABLE_TARGETS; + if (value === undefined || value === null) + return appUid === null ? SESSION_TARGETS : DEFAULT_DURABLE_TARGETS; if (!Array.isArray(value) || value.length === 0) throw badRequest( 'targets must be a non-empty array', @@ -494,6 +629,11 @@ const parseTargets = ( 'A `single` subscription needs a `worker` target and may not target `push`', 'invalid_targets', ); + if (appUid === null && targets.includes('worker')) + throw badRequest( + 'A subscription with no app has no events worker to target', + 'invalid_targets', + ); return targets; }; @@ -529,6 +669,25 @@ const parseHandlerName = (value: unknown): string | null => { return value; }; +/** Hex digest of the inline source a subscribe claims it is binding. */ +const parseHandlerHash = (value: unknown): string | null => { + if (value === undefined || value === null) return null; + if (typeof value !== 'string' || !/^[0-9a-f]{64}$/.test(value)) + throw badRequest( + 'handlerHash must be a sha-256 hex digest', + 'bad_request', + ); + return value; +}; + +/** An app uid a user session names, for a surface with no app of its own. */ +const parseAppUid = (value: unknown): string | null => { + if (value === undefined || value === null || value === '') return null; + if (typeof value !== 'string') + throw badRequest('appUid must be a string', 'bad_request'); + return value; +}; + /** Stored as JSON text; the byte cap is the store's to enforce. */ const parseContext = (value: unknown): string | null => { if (value === undefined || value === null) return null; @@ -804,23 +963,27 @@ export class EventsService extends PuterService { await this.#spendCallBudget(holderUserId); const delivery = parseDelivery(request?.delivery); - const targets = parseTargets(request?.targets, delivery); + const appUid = actor.effectiveApp?.uid ?? null; + const targets = parseTargets(request?.targets, delivery, appUid); const handlerName = parseHandlerName(request?.handlerName); + const handlerHash = parseHandlerHash(request?.handlerHash); const context = parseContext(request?.context); const expiresAt = parseExpiresAt(request?.expiresAt); // A `single` is owed to exactly one consumer, and the handler is the - // only one that is always there to take it. Whether the handler exists - // is the publish surface's question, not this one's. + // only one that is always there to take it. if (delivery === 'single' && !handlerName) throw handlerRequired(); + if (handlerName) + await this.#assertHandlerBinding(appUid, handlerName, handlerHash); + const rawSubject = String(request?.subject ?? ''); const anchor = await this.#resolveSubscribeAnchor(actor, rawSubject); const { row, bump } = await this.stores.durableSubscription.create({ holderUserId, ownerUserId: anchor.ownerUserId, - appUid: actor.effectiveApp?.uid ?? null, + appUid, subject: anchor.subject, token: anchor.token, anchorUid: anchor.uid, @@ -939,6 +1102,220 @@ export class EventsService extends PuterService { await this.#drain(row); } + // -- Handlers ---------------------------------------------------- + + /** + * Publish one named handler for an app. + * + * The name is the identity and the hash is only a change detector, so + * re-publishing the same source is a no-op and re-publishing a name that is + * suspending subscriptions brings them back. A publish whose base has moved + * under it — two build steps racing — is refused rather than resolved. + */ + async publishHandler( + actor: Actor, + request: PublishHandlerRequest, + ): Promise { + await this.#spendHandlerBudget(actor); + const appUid = await this.#handlerApp(actor, request?.appUid); + return this.#publishOne(appUid, request); + } + + /** + * Publish a set of handlers, which is what a build step has. Each item is + * the same operation under the same rules; one that is refused stops the + * pass, so a deploy either lands its set or reports which name it stopped + * at rather than leaving half a build published under a success. + */ + async publishHandlers( + actor: Actor, + request: PublishHandlersRequest, + ): Promise { + await this.#spendHandlerBudget(actor); + const appUid = await this.#handlerApp(actor, request?.appUid); + const handlers = request?.handlers; + if (!Array.isArray(handlers) || handlers.length === 0) + throw badRequest( + 'handlers must be a non-empty array', + 'bad_request', + ); + if (handlers.length > EVENTS_HANDLER_PUBLISH_BATCH) + throw badRequest( + `handlers may not exceed ${EVENTS_HANDLER_PUBLISH_BATCH} entries`, + 'bad_request', + ); + + const published: PublishedHandlerView[] = []; + for (const item of handlers) { + if (!item || typeof item !== 'object' || Array.isArray(item)) + throw badRequest( + 'each handler must be an object', + 'bad_request', + ); + published.push( + await this.#publishOne(appUid, item as PublishHandlerRequest), + ); + } + return published; + } + + /** + * What an app has published, and how many subscriptions each name carries. + * Never the source: a listing is the one handler surface that can be called + * repeatedly, and the source is the app's own code. + */ + async listHandlers( + actor: Actor, + request: HandlerScopeRequest = {}, + ): Promise { + return this.stores.eventHandler.listForApp( + await this.#handlerApp(actor, request?.appUid), + ); + } + + /** + * Take a name out of service. With nothing bound to it the row simply goes; + * with dependents it goes too, and they suspend with `handler_not_found` + * rather than being deleted — republishing the name is what brings them + * back, so a bad deploy is recoverable and a rename is deliberately not. + */ + async removeHandler( + actor: Actor, + request: HandlerNameRequest, + ): Promise { + await this.#spendHandlerBudget(actor); + const appUid = await this.#handlerApp(actor, request?.appUid); + const name = parseHandlerName(request?.name); + if (!name) throw badRequest('name is required', 'bad_request'); + + const removed = await this.stores.eventHandler.remove(appUid, name); + const suspended = await this.#suspendHandlerDependents(appUid, name); + return { name, removed: removed !== null, suspended }; + } + + // -- Suspension -------------------------------------------------- + + /** + * Take rows out of service without deleting them, and do to their backlogs + * whatever the reason says. Every suspension goes through here so the + * backlog policy cannot be forgotten at one call site: a suspended + * subscription stops being metered, and one that stops being metered while + * holding a full backlog is a hold nobody pays for. + */ + async suspendSubscriptions( + rows: readonly DurableSubscription[], + reason: SuspendedReason, + ): Promise { + return (await this.#suspend(rows, reason)).length; + } + + /** + * The rows this pass was the one to suspend. An unshare withdraws several + * grant strings in a row and every one of them settles, so a row another + * pass already took is not this pass's to purge or announce. + */ + async #suspend( + rows: readonly DurableSubscription[], + reason: SuspendedReason, + ): Promise { + if (rows.length === 0) return []; + + const { suspended, bumps } = + await this.stores.durableSubscription.suspend(rows, reason); + const policy = backlogPolicyFor(reason); + for (const row of suspended) { + try { + if (policy.cap === 0) + await this.stores.pendingDelivery.purge(row.subId); + else { + const shed = await this.stores.pendingDelivery.hold( + row.subId, + policy.cap, + policy.ttlMs, + ); + if (shed) this.#reportShed([shed]); + } + } catch (err) { + console.warn( + '[events] could not settle the backlog of a suspended subscription', + row.subId, + err, + ); + } + // Takes any coalesced delivery with it: one still in flight is for + // a subscription that has stopped. + this.#forget(row.subId); + } + for (const bump of bumps) this.#publishGeneration(bump, true); + return suspended; + } + + /** + * Put rows back in service, and hand over what survived their hold. Rows + * suspended for a reason that never lifts are skipped: consent to watch is + * re-established by subscribing again, never by resuming. + */ + async resumeSubscriptions( + rows: readonly DurableSubscription[], + ): Promise { + const resuming = rows.filter( + (row) => + row.suspendedAt !== null && + isSuspendedReason(row.suspendedReason) && + isResumable(row.suspendedReason), + ); + if (resuming.length === 0) return 0; + + const bumps = await this.stores.durableSubscription.resume(resuming); + for (const row of resuming) { + try { + await this.stores.pendingDelivery.releaseHold(row.subId); + await this.#drain(row); + } catch (err) { + console.warn( + '[events] could not hand over the backlog of a resumed subscription', + row.subId, + err, + ); + } + } + for (const bump of bumps) this.#publishGeneration(bump, true); + return resuming.length; + } + + /** + * Stop one subscription after its handler kept failing. The counting is the + * retry path's; this is the state it drives, and the developer is told + * because it is their code that stopped working. + */ + async suspendForFailures(subId: string): Promise { + return this.#suspendOne(subId, 'failures', { notifyDeveloper: true }); + } + + /** + * Stop one subscription whose holder cannot pay for it. The 402 is the + * metering path's to raise; the backlog is held on the short window, + * because the resume condition is usually a top-up minutes away. + */ + async suspendForNoCredit(subId: string): Promise { + return this.#suspendOne(subId, 'no_credit', { notifyDeveloper: false }); + } + + /** + * Put back what a restored balance releases. The seam credit restoration + * calls: one holder's rows, one pass, and nothing else has to know how a + * suspension is spelled. + */ + async resumeForCredit(holderUserId: number): Promise { + if (!this.enabled) return 0; + return this.resumeSubscriptions( + await this.stores.durableSubscription.listSuspendedForHolder( + holderUserId, + 'no_credit', + ), + ); + } + /** * Drop rows past their expiry, in batches, and report how many went. Every * node sweeps; the delete is idempotent, so two overlapping costs a few @@ -1011,16 +1388,17 @@ export class EventsService extends PuterService { await this.stores.pendingDelivery.purge(subId); continue; } - // A suspended row keeps what it is owed — what happens to that - // backlog is the suspension's decision, not the sweeper's. It - // goes to the back of the line so it cannot hold the head. + // A suspended row is not delivered to, but the backlog its + // suspension put a deadline on is this pass's to enforce: past + // it the events go and a gap marker says so, which is the half + // that keeps a suspension from being a free memory hold. if (row.suspendedAt !== null) { // Except a revoked row's, which names paths its holder may // no longer see: anything a dispatch in flight queued after - // the settle's own purge goes now, not at the reap. + // the settle's own purge goes now, not at a deadline. if (row.suspendedReason === 'permission_revoked') await this.stores.pendingDelivery.purge(subId); - else await this.stores.pendingDelivery.defer(subId); + else await this.stores.pendingDelivery.expireHold(subId); continue; } attempted += await this.#drain(row); @@ -1036,6 +1414,200 @@ export class EventsService extends PuterService { return row.expiresAt !== null && row.expiresAt <= Date.now() / 1000; } + // -- Handler internals ------------------------------------------- + + /** + * Which app's handlers this call is about, and whether the caller may + * deploy them. Publishing is a developer operation: the app token's own app + * or the one a user session named, and in both cases an app that user + * owns. + * + * An app that is not there answers the same as one the caller does not own + * — which apps exist is not this surface's to disclose. + */ + async #handlerApp(actor: Actor, requested: unknown): Promise { + if (!this.enabled) throw disabled(); + const userId = actor.user?.id; + if (userId === undefined) throw disabled(); + + const acting = actor.effectiveApp; + // Unresolved is not "no app": reading it that way is what would let an + // app token publish into a namespace it never named. + if (acting === undefined) throw handlerAppForbidden(); + + const named = parseAppUid(requested); + if (acting && named !== null && named !== acting.uid) + throw handlerAppForbidden(); + + const appUid = acting?.uid ?? named; + if (!appUid) throw handlerAppRequired(); + + const app = await this.stores.app.getByUid(appUid); + if (!app) throw handlerAppForbidden(); + if ( + Number((app as { owner_user_id?: unknown }).owner_user_id) !== + Number(userId) + ) + throw handlerAppForbidden(); + return appUid; + } + + async #spendHandlerBudget(actor: Actor): Promise { + const userId = actor.user?.id; + if (userId === undefined) throw disabled(); + const ok = await checkRateLimit( + `${EVENTS_HANDLER_PUBLISH_LIMIT.scope}:${userId}`, + EVENTS_HANDLER_PUBLISH_LIMIT.limit, + EVENTS_HANDLER_PUBLISH_LIMIT.window, + ); + if (!ok) throw tooManyCalls(); + } + + /** One publish, and whatever it releases. */ + async #publishOne( + appUid: string, + item: PublishHandlerRequest, + ): Promise { + const { handler, outcome } = await this.stores.eventHandler.publish({ + appUid, + name: String(item?.name ?? ''), + source: typeof item?.source === 'string' ? item.source : '', + ifHash: typeof item?.ifHash === 'string' ? item.ifHash : null, + replace: item?.replace === true, + }); + + return { + name: handler.name, + hash: handler.sourceHash, + updatedAt: handler.updatedAt, + outcome, + resumed: await this.#resumeHandlerDependents(appUid, handler.name), + }; + } + + /** + * Suspend everything bound to a name that is no longer published, in + * batches so a widely-used handler cannot make one call hold the whole + * set. + */ + async #suspendHandlerDependents( + appUid: string, + name: string, + ): Promise { + let suspended = 0; + for (;;) { + const batch = await this.stores.durableSubscription.listByHandler( + appUid, + name, + ); + if (batch.length === 0) break; + suspended += await this.suspendSubscriptions( + batch, + 'handler_not_found', + ); + if (batch.length < HANDLER_SETTLE_BATCH) break; + } + if (suspended > 0) await this.#notifySuspended(appUid, name, suspended); + return suspended; + } + + /** Bring back what was waiting on this name. The other half of a removal. */ + async #resumeHandlerDependents( + appUid: string, + name: string, + ): Promise { + let resumed = 0; + for (;;) { + const batch = await this.stores.durableSubscription.listByHandler( + appUid, + name, + { suspendedReason: 'handler_not_found' }, + ); + if (batch.length === 0) break; + resumed += await this.resumeSubscriptions(batch); + if (batch.length < HANDLER_SETTLE_BATCH) break; + } + return resumed; + } + + /** One row into a suspended state, for the reasons a single row reaches. */ + async #suspendOne( + subId: string, + reason: SuspendedReason, + options: { notifyDeveloper: boolean }, + ): Promise { + if (!this.enabled) return false; + const row = await this.stores.durableSubscription.getBySubId(subId); + if (!row || row.suspendedAt !== null) return false; + + await this.suspendSubscriptions([row], reason); + if (options.notifyDeveloper && row.appUid && row.handlerName) + await this.#notifySuspended(row.appUid, row.handlerName, 1); + return true; + } + + /** + * Tell an app's developer that subscriptions on one of their handlers have + * stopped. Theirs rather than the holder's: the handler is the developer's + * code, and the fix is a publish only they can make. + */ + async #notifySuspended( + appUid: string, + handlerName: string, + subscriptions: number, + ): Promise { + try { + const app = await this.stores.app.getByUid(appUid); + const ownerUserId = Number( + (app as { owner_user_id?: unknown } | null)?.owner_user_id, + ); + if (!Number.isFinite(ownerUserId) || ownerUserId <= 0) return; + + await this.services.notification.notify( + [ownerUserId], + { + title: 'Event subscriptions were suspended', + handler: handlerName, + subscriptions, + }, + { type: 'app.events.suspended', appUid }, + ); + } catch (err) { + console.warn( + '[events] could not report a suspended handler', + handlerName, + err, + ); + } + } + + /** + * Whether a subscription may bind the handler name it asked for. + * + * Handlers are published per app, so an account-scoped row has no namespace + * to bind in — the name is stored and binds nothing. An inline body is + * different: sending a hash _is_ the binding claim, and there is nothing + * for it to match. + */ + async #assertHandlerBinding( + appUid: string | null, + name: string, + hash: string | null, + ): Promise { + if (appUid === null) { + if (hash !== null) throw handlerNotFound(name); + return; + } + + const published = await this.stores.eventHandler.getByName( + appUid, + name, + ); + if (!published) throw handlerNotFound(name); + if (hash !== null && hash !== published.sourceHash) + throw handlerHashMismatch(name); + } + /** * Resolve, authorize and compile one subscribe request. Shared so a durable * row cannot be created under a weaker check than a session one. @@ -1618,25 +2190,11 @@ export class EventsService extends PuterService { : await this.#leftUnauthorized(held, revocation.permission); if (settling.length === 0) return 0; - // Only the rows this pass was the one to suspend are its to purge and - // announce: an unshare withdraws several grant strings in a row, and - // every one of them runs this settle. - const { suspended, bumps } = - await this.stores.durableSubscription.suspend( - settling, - 'permission_revoked', - ); - for (const row of suspended) { - // Unlike every other suspension, this backlog goes at once: it - // holds the paths of a resource its holder has just lost the right - // to see, and keeping it for a resume that by design never comes - // turns a revocation into a delayed disclosure. - await this.stores.pendingDelivery.purge(row.subId).catch(() => {}); - // Takes the coalesced deliveries with it: one still in flight names - // exactly what its holder has stopped being allowed to see. - this.#forget(row.subId); - } - for (const bump of bumps) this.#publishGeneration(bump, true); + // The `permission_revoked` arm of the shared policy purges the backlog + // at once: it holds the paths of a resource its holder has just lost + // the right to see, and keeping it for a resume that by design never + // comes turns a revocation into a delayed disclosure. + const suspended = await this.#suspend(settling, 'permission_revoked'); await this.#notifyEnded(suspended, 'permission_revoked'); return suspended.length; } diff --git a/src/backend/services/events/durable.integration.test.ts b/src/backend/services/events/durable.integration.test.ts index 8ab39c0af..c47d3f5b4 100644 --- a/src/backend/services/events/durable.integration.test.ts +++ b/src/backend/services/events/durable.integration.test.ts @@ -214,7 +214,9 @@ describe('creating a durable subscription over HTTP', () => { expect(created.body).toMatchObject({ subject: `fs:${anchor}`, delivery: 'broadcast', - targets: ['socket', 'worker'], + // No app, so no events worker to target — see the null-app + // suite below. + targets: ['socket'], appUid: null, }); expect(created.body.context).toBeUndefined(); @@ -231,7 +233,7 @@ describe('creating a durable subscription over HTTP', () => { expect(created.body).toMatchObject({ delivery: 'single', handlerName: 'onWrite', - targets: ['socket', 'worker'], + targets: ['socket'], }); }); @@ -255,6 +257,21 @@ describe('creating a durable subscription over HTTP', () => { expect(refused.body.code).toBe('invalid_targets'); }); + it('refuses a `worker` target from an account session naming no app', async () => { + // Exactly one events worker per app: an account session has none, so + // asking for the worker target explicitly is refused rather than + // silently dropped — silently dropping it would leave the caller + // thinking background delivery was configured when it never could be. + const refused = await subscribe(env.users.user.token, { + delivery: 'single', + handlerName: 'onWrite', + targets: ['socket', 'worker'], + }); + + expect(refused.status).toBe(400); + expect(refused.body.code).toBe('invalid_targets'); + }); + it('refuses a target outside the known set', async () => { const refused = await subscribe(env.users.user.token, { targets: ['socket', 'carrier-pigeon'], diff --git a/src/backend/services/events/handlers.integration.test.ts b/src/backend/services/events/handlers.integration.test.ts new file mode 100644 index 000000000..383c561c6 --- /dev/null +++ b/src/backend/services/events/handlers.integration.test.ts @@ -0,0 +1,769 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Handlers end to end: who may publish them, what a subscription binding one is + * held to, and the full removal-suspends / republish-resumes cycle. + * + * The context assertions belong here rather than in a store test because the + * point is what crosses each boundary: a listing never carries values, and one + * shared handler delivers each subscriber their own. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { v4 as uuidv4 } from 'uuid'; +import { + EVENTS_COALESCE_WINDOW_MS, + EVENTS_SUSPENDED_PENDING_CAP, +} from '../../controllers/events/limits.js'; +import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js'; +import { hashContent } from '../../stores/events/EventHandlerStore.js'; +import type { IConfig } from '../../types.js'; +import type { DurableSubscriptionView } from './EventsService.js'; +import { RecordingWorkerInvoker } from './workerSeam.js'; + +const BOOT_TIMEOUT_MS = 120_000; + +const SOURCE = 'async ({ event, ctx }) => { await fetch(ctx.url, { method: "POST" }); }'; +const NEXT_SOURCE = 'async ({ event, ctx }) => { console.log(event.path, ctx.url); }'; + +let env: PuterTestEnv; +let userId: number; +let otherUserId: number; +let anchor: string; +let otherAnchor: string; +let appUid: string; +let appToken: string; +let otherAppToken: string; +let foreignAppUid: string; +let foreignAppToken: string; +let worker: RecordingWorkerInvoker; + +const events = () => env.server.services.events; +const durable = () => env.server.stores.durableSubscription; +const pending = () => env.server.stores.pendingDelivery; + +interface ApiResponse { + status: number; + body: Record; +} + +const call = async ( + method: 'GET' | 'POST', + path: string, + token: string, + body?: object, +): Promise => { + const response = await fetch(new URL(path, env.apiOrigin), { + method, + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${token}`, + }, + ...(body ? { body: JSON.stringify(body) } : {}), + }); + return { + status: response.status, + body: (await response.json()) as Record, + }; +}; + +const publish = (token: string, body: object): Promise => + call('POST', '/events/handlers/publish', token, body); + +const seedHandler = async ( + name = 'ingestUpload', + source = SOURCE, +): Promise => { + const published = await publish(appToken, { name, source }); + expect(published.status).toBe(200); + return String(published.body.hash); +}; + +/** + * An app owned by `ownerUserId`, granted `list` on each named user's anchor so + * its token can subscribe there. + */ +const makeApp = async ( + ownerUserId: number, + grants: Array<{ token: string; path: string }>, +): Promise<{ uid: string; tokens: string[] }> => { + const uid = `app-${uuidv4()}`; + await env.server.clients.db.write( + 'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)', + [uid, uid, uid, `https://${uid}.example/`, ownerUserId], + ); + + const tokens: string[] = []; + for (const grant of grants) { + const entry = await env.server.stores.fsEntry.getEntryByPath(grant.path); + const { actor } = await env.server.services.auth.authenticate(grant.token); + await env.server.services.permission.grantUserAppPermission( + actor!, + uid, + `fs:${entry!.uid}:list`, + ); + tokens.push( + await env.server.services.auth.getUserAppToken(actor!, uid), + ); + } + return { uid, tokens }; +}; + +const subscribe = (token: string, body: object): Promise => + call('POST', '/events/subscribe', token, body); + +const listSubscriptions = async ( + token: string, +): Promise => + (await call('GET', '/events/subscriptions', token)) + .body.items as DurableSubscriptionView[]; + +const rowOf = (subId: string) => durable().getBySubId(subId); + +const touch = (holder: number, path: string) => + env.server.services.fs.touch(holder, { path }); + +beforeAll(async () => { + env = await setupPuterTestEnv({ events: { enabled: true } } as IConfig); + + const user = await env.server.stores.user.getByUsername( + env.users.user.username, + ); + userId = user!.id; + const other = await env.server.stores.user.getByUsername( + env.users.other.username, + ); + otherUserId = other!.id; + + anchor = `/${env.users.user.username}/handlers`; + otherAnchor = `/${env.users.other.username}/handlers`; + await env.server.services.fs.mkdir(userId, { + path: anchor, + createMissingParents: true, + }); + await env.server.services.fs.mkdir(otherUserId, { + path: otherAnchor, + createMissingParents: true, + }); + + // One app, owned by `user`, that both accounts have let in. + const owned = await makeApp(userId, [ + { token: env.users.user.token, path: anchor }, + { token: env.users.other.token, path: otherAnchor }, + ]); + appUid = owned.uid; + [appToken, otherAppToken] = owned.tokens; + + // A second app, owned by somebody else, whose token `user` also holds. + const foreign = await makeApp(otherUserId, [ + { token: env.users.user.token, path: anchor }, + ]); + foreignAppUid = foreign.uid; + [foreignAppToken] = foreign.tokens; + + worker = new RecordingWorkerInvoker(); + events().worker = worker; +}, BOOT_TIMEOUT_MS); + +afterAll(async () => { + await env?.shutdown(); +}); + +beforeEach(async () => { + await env.server.clients.db.write('DELETE FROM `event_handlers`', []); + for (const row of await durable().listActiveForHolder(userId, null)) + await durable().remove(row); + await env.server.clients.db.write('DELETE FROM `event_subscriptions`', []); + await env.server.stores.eventSubscription.rebuildDurable(userId, []); + await env.server.stores.eventSubscription.rebuildDurable(otherUserId, []); + worker.recorded.length = 0; +}); + +describe('who may publish a handler', () => { + it('lets an app token publish into its own app', async () => { + const published = await publish(appToken, { + name: 'ingestUpload', + source: SOURCE, + }); + + expect(published.status).toBe(200); + expect(published.body).toMatchObject({ + name: 'ingestUpload', + hash: hashContent(SOURCE), + outcome: 'created', + resumed: 0, + }); + expect(published.body.source).toBeUndefined(); + }); + + it('lets an account session publish by naming an app it owns', async () => { + const published = await publish(env.users.user.token, { + appUid, + name: 'ingestUpload', + source: SOURCE, + }); + expect(published.status).toBe(200); + }); + + it('refuses an account session that names no app', async () => { + const refused = await publish(env.users.user.token, { + name: 'ingestUpload', + source: SOURCE, + }); + + expect(refused.status).toBe(400); + expect(refused.body.code).toBe('events_handler_app_required'); + }); + + it('refuses an app token whose app its user does not own', async () => { + const refused = await publish(foreignAppToken, { + name: 'ingestUpload', + source: SOURCE, + }); + + expect(refused.status).toBe(403); + expect(refused.body.code).toBe('events_handler_forbidden'); + }); + + it('refuses an account session naming an app it does not own', async () => { + const refused = await publish(env.users.user.token, { + appUid: foreignAppUid, + name: 'ingestUpload', + source: SOURCE, + }); + + expect(refused.status).toBe(403); + expect(refused.body.code).toBe('events_handler_forbidden'); + }); + + it('refuses an app token reaching into another app`s namespace', async () => { + const refused = await publish(appToken, { + appUid: foreignAppUid, + name: 'ingestUpload', + source: SOURCE, + }); + + expect(refused.status).toBe(403); + expect(refused.body.code).toBe('events_handler_forbidden'); + }); +}); + +describe('publishing a set', () => { + it('takes a build step`s handlers in one call', async () => { + const published = await call( + 'POST', + '/events/handlers/publishAll', + appToken, + { + handlers: [ + { name: 'ingestUpload', source: SOURCE }, + { name: 'indexDocument', source: NEXT_SOURCE }, + ], + }, + ); + + expect(published.status).toBe(200); + expect( + (published.body.handlers as Array<{ name: string }>).map( + (row) => row.name, + ), + ).toEqual(['ingestUpload', 'indexDocument']); + }); + + it('stops at the item it cannot publish rather than reporting success', async () => { + await seedHandler(); + + const refused = await call( + 'POST', + '/events/handlers/publishAll', + appToken, + { + handlers: [ + { name: 'indexDocument', source: NEXT_SOURCE }, + { name: 'ingestUpload', source: NEXT_SOURCE }, + ], + }, + ); + + expect(refused.status).toBe(409); + expect(refused.body.code).toBe('events_handler_conflict'); + // The item before the conflict landed; the caller is told where it + // stopped rather than being left to guess. + const listed = await call('GET', '/events/handlers/list', appToken); + expect( + (listed.body.handlers as Array<{ name: string }>).map((h) => h.name), + ).toContain('indexDocument'); + }); +}); + +describe('listing handlers', () => { + it('reports names and hashes and never the source', async () => { + await seedHandler(); + + const listed = await call('GET', '/events/handlers/list', appToken); + + expect(listed.status).toBe(200); + expect(listed.body.handlers).toEqual([ + { + name: 'ingestUpload', + hash: hashContent(SOURCE), + updatedAt: expect.any(Number), + subscriptions: 0, + }, + ]); + }); +}); + +describe('binding a subscription to a handler', () => { + it('binds a name the app has published', async () => { + await seedHandler(); + + const created = await subscribe(appToken, { + subject: `fs:${anchor}`, + delivery: 'single', + handlerName: 'ingestUpload', + targets: ['worker'], + }); + + expect(created.status).toBe(200); + expect(created.body.handlerName).toBe('ingestUpload'); + }); + + it('refuses a name the app never published', async () => { + const refused = await subscribe(appToken, { + subject: `fs:${anchor}`, + delivery: 'single', + handlerName: 'ingestUpload', + targets: ['worker'], + }); + + expect(refused.status).toBe(404); + expect(refused.body.code).toBe('events_handler_not_found'); + expect(await durable().listActiveForHolder(userId, appUid)).toEqual([]); + }); + + it('binds an inline body whose hash is what is published', async () => { + const hash = await seedHandler(); + + const created = await subscribe(appToken, { + subject: `fs:${anchor}`, + delivery: 'single', + handlerName: 'ingestUpload', + handlerHash: hash, + targets: ['worker'], + }); + + expect(created.status).toBe(200); + }); + + it('refuses an inline body that is not what is published', async () => { + await seedHandler(); + + const refused = await subscribe(appToken, { + subject: `fs:${anchor}`, + delivery: 'single', + handlerName: 'ingestUpload', + handlerHash: hashContent(NEXT_SOURCE), + targets: ['worker'], + }); + + expect(refused.status).toBe(409); + expect(refused.body.code).toBe('events_handler_hash_mismatch'); + expect(await durable().listActiveForHolder(userId, appUid)).toEqual([]); + }); + + it('still needs a name for a subscription owed to one consumer', async () => { + const refused = await subscribe(appToken, { + subject: `fs:${anchor}`, + delivery: 'single', + targets: ['worker'], + }); + + expect(refused.status).toBe(400); + expect(refused.body.code).toBe('events_handler_required'); + }); +}); + +describe('the handler lifecycle', () => { + const bind = async (): Promise => { + await seedHandler(); + const created = await subscribe(appToken, { + subject: `fs:${anchor}`, + delivery: 'single', + handlerName: 'ingestUpload', + targets: ['worker'], + }); + expect(created.status).toBe(200); + return String(created.body.subId); + }; + + it('deletes outright a name nothing is bound to', async () => { + await seedHandler(); + + const removed = await call( + 'POST', + '/events/handlers/remove', + appToken, + { name: 'ingestUpload' }, + ); + + expect(removed.body).toMatchObject({ removed: true, suspended: 0 }); + const listed = await call('GET', '/events/handlers/list', appToken); + expect(listed.body.handlers).toEqual([]); + }); + + it('suspends dependents, then resumes them when the name comes back', async () => { + const subId = await bind(); + + const removed = await call( + 'POST', + '/events/handlers/remove', + appToken, + { name: 'ingestUpload' }, + ); + expect(removed.body).toMatchObject({ removed: true, suspended: 1 }); + + const suspended = await rowOf(subId); + expect(suspended).toMatchObject({ + suspendedReason: 'handler_not_found', + }); + expect(suspended!.suspendedAt).toBeGreaterThan(0); + // Out of every watched set, so no event under the anchor reaches it. + expect( + await env.server.stores.eventSubscription.getForTokens(userId, [ + suspended!.token, + ]), + ).toEqual([]); + + const republished = await publish(appToken, { + name: 'ingestUpload', + source: NEXT_SOURCE, + }); + expect(republished.body).toMatchObject({ + outcome: 'created', + resumed: 1, + }); + + const resumed = await rowOf(subId); + expect(resumed).toMatchObject({ + suspendedAt: null, + suspendedReason: null, + }); + // Back in the watched set, and the generation moved so every other + // region rebuilds rather than staying blind to it. + expect( + ( + await env.server.stores.eventSubscription.getForTokens(userId, [ + resumed!.token, + ]) + ).map((row) => row.subId), + ).toEqual([subId]); + }); + + it('shows the suspension and its reason in the holder`s listing', async () => { + const subId = await bind(); + await call('POST', '/events/handlers/remove', appToken, { + name: 'ingestUpload', + }); + + const listed = await listSubscriptions(appToken); + expect(listed.find((row) => row.subId === subId)).toMatchObject({ + suspendedReason: 'handler_not_found', + }); + }); + + it('counts a suspended dependent as still bound to the name', async () => { + await bind(); + await call('POST', '/events/handlers/remove', appToken, { + name: 'ingestUpload', + }); + await publish(appToken, { name: 'ingestUpload', source: NEXT_SOURCE }); + + const listed = await call('GET', '/events/handlers/list', appToken); + expect(listed.body.handlers).toEqual([ + expect.objectContaining({ name: 'ingestUpload', subscriptions: 1 }), + ]); + }); + + it('trims the backlog a suspension holds and drops it when the hold lapses', async () => { + const subId = await bind(); + + for (let i = 0; i < EVENTS_SUSPENDED_PENDING_CAP + 5; i++) { + await pending().enqueue(subId, { + id: `event-${i}`, + subject: `fs:${anchor}`, + op: 'write', + uid: 'node', + path: `${anchor}/f-${i}.txt`, + self: true, + ts: Date.now(), + seq: 0, + }); + } + expect(await pending().depth(subId)).toBeGreaterThan( + EVENTS_SUSPENDED_PENDING_CAP, + ); + + await call('POST', '/events/handlers/remove', appToken, { + name: 'ingestUpload', + }); + + // Trimmed to the reduced cap, with one gap marker taking the place of + // what went. + expect(await pending().depth(subId)).toBe(EVENTS_SUSPENDED_PENDING_CAP); + + // The sweeper only enforces the deadline once it has passed. + await events().sweepPending(); + expect(await pending().depth(subId)).toBe(EVENTS_SUSPENDED_PENDING_CAP); + + await pending().hold(subId, EVENTS_SUSPENDED_PENDING_CAP, -1); + await events().sweepPending(); + + // Everything held went, and one marker says so rather than the + // subscription reading the silence as "nothing happened". + expect(await pending().depth(subId)).toBe(1); + const claimed = await pending().claim(subId); + expect(claimed?.event).toMatchObject({ + op: 'gap', + reason: 'suspended_backlog_expired', + }); + }); + + it('hands over the backlog a hold kept once the handler is republished', async () => { + const subId = await bind(); + + await pending().enqueue(subId, { + id: 'held-1', + subject: `fs:${anchor}`, + op: 'write', + uid: 'node', + path: `${anchor}/held.txt`, + self: true, + ts: Date.now(), + seq: 0, + }); + + await call('POST', '/events/handlers/remove', appToken, { + name: 'ingestUpload', + }); + // Under the reduced cap, so held rather than dropped. + expect(await pending().depth(subId)).toBe(1); + + await publish(appToken, { name: 'ingestUpload', source: NEXT_SOURCE }); + + // `releaseHold` lifts the reduced cap and `resumeSubscriptions` drains + // what survived it — the held event reaches the handler rather than + // sitting there until something else asks for it. (The recording + // stub never reports `settled`, so the entry's lease stays open + // rather than the depth dropping to zero — that half is covered by + // `singleDelivery.test.ts`'s settle case.) + await vi.waitFor( + () => + expect( + worker.recorded.some( + (call) => + call.subId === subId && + (call.event as { id?: string }).id === 'held-1', + ), + ).toBe(true), + { timeout: EVENTS_COALESCE_WINDOW_MS * 20, interval: 25 }, + ); + }); + + it('delivers again end to end once a republish resumes it', async () => { + const subId = await bind(); + + await call('POST', '/events/handlers/remove', appToken, { + name: 'ingestUpload', + }); + expect(await rowOf(subId)).toMatchObject({ + suspendedReason: 'handler_not_found', + }); + + const republished = await publish(appToken, { + name: 'ingestUpload', + source: NEXT_SOURCE, + }); + expect(republished.body).toMatchObject({ resumed: 1 }); + expect(await rowOf(subId)).toMatchObject({ suspendedAt: null }); + + // Tokens re-cached and the generation moved, per the test above; the + // full cycle also means a *new* write reaches the handler again, + // exactly as it would have before the handler was ever removed. + await touch(userId, `${anchor}/after-resume.txt`); + + await vi.waitFor( + () => + expect( + worker.recorded.some((call) => call.subId === subId), + ).toBe(true), + { timeout: EVENTS_COALESCE_WINDOW_MS * 20, interval: 25 }, + ); + }); + + it('does not resume a subscription a withdrawn grant stopped', async () => { + const subId = await bind(); + const row = await rowOf(subId); + await events().suspendSubscriptions([row!], 'permission_revoked'); + + await call('POST', '/events/handlers/remove', appToken, { + name: 'ingestUpload', + }); + const republished = await publish(appToken, { + name: 'ingestUpload', + source: SOURCE, + }); + + expect(republished.body.resumed).toBe(0); + expect(await rowOf(subId)).toMatchObject({ + suspendedReason: 'permission_revoked', + }); + }); + + it('purges the backlog of a withdrawn grant instead of holding it', async () => { + const subId = await bind(); + await pending().enqueue(subId, { + id: 'event-1', + subject: `fs:${anchor}`, + op: 'write', + uid: 'node', + path: `${anchor}/f.txt`, + self: true, + ts: Date.now(), + seq: 0, + }); + + const row = await rowOf(subId); + await events().suspendSubscriptions([row!], 'permission_revoked'); + + expect(await pending().depth(subId)).toBe(0); + }); + + it('suspends and resumes one row for the reasons the delivery path raises', async () => { + const subId = await bind(); + + expect(await events().suspendForNoCredit(subId)).toBe(true); + expect(await rowOf(subId)).toMatchObject({ + suspendedReason: 'no_credit', + }); + // Already out of service, so nothing to do a second time. + expect(await events().suspendForNoCredit(subId)).toBe(false); + + expect(await events().resumeForCredit(userId)).toBe(1); + expect(await rowOf(subId)).toMatchObject({ suspendedAt: null }); + + expect(await events().suspendForFailures(subId)).toBe(true); + expect(await rowOf(subId)).toMatchObject({ + suspendedReason: 'failures', + }); + // A credit restore does not lift a suspension it did not cause. + expect(await events().resumeForCredit(userId)).toBe(0); + }); +}); + +describe('the context a subscription carries', () => { + it('reports its key names and a hash, never its values', async () => { + await seedHandler(); + const created = await subscribe(appToken, { + subject: `fs:${anchor}`, + delivery: 'single', + handlerName: 'ingestUpload', + targets: ['worker'], + context: { url: 'https://ingest.example/secret-token', retries: 3 }, + }); + expect(created.status).toBe(200); + + const [listed] = await listSubscriptions(appToken); + expect(listed.contextKeys).toEqual(['retries', 'url']); + expect(listed.contextHash).toMatch(/^[0-9a-f]{64}$/); + expect(JSON.stringify(listed)).not.toContain('secret-token'); + }); + + it('has neither for a subscription that carries none', async () => { + await seedHandler(); + await subscribe(appToken, { + subject: `fs:${anchor}`, + delivery: 'single', + handlerName: 'ingestUpload', + targets: ['worker'], + }); + + const [listed] = await listSubscriptions(appToken); + expect(listed.contextKeys).toBeNull(); + expect(listed.contextHash).toBeNull(); + }); + + it('refuses one past the cap', async () => { + await seedHandler(); + const refused = await subscribe(appToken, { + subject: `fs:${anchor}`, + delivery: 'single', + handlerName: 'ingestUpload', + targets: ['worker'], + context: { blob: 'x'.repeat(5000) }, + }); + + expect(refused.status).toBe(413); + expect(refused.body.code).toBe('events_context_too_large'); + }); + + it('delivers each subscriber their own against one shared handler', async () => { + await seedHandler(); + + const mine = await subscribe(appToken, { + subject: `fs:${anchor}`, + delivery: 'single', + handlerName: 'ingestUpload', + targets: ['worker'], + context: { url: 'https://mine.example' }, + }); + const theirs = await subscribe(otherAppToken, { + subject: `fs:${otherAnchor}`, + delivery: 'single', + handlerName: 'ingestUpload', + targets: ['worker'], + context: { url: 'https://theirs.example' }, + }); + expect(mine.status).toBe(200); + expect(theirs.status).toBe(200); + + await touch(userId, `${anchor}/mine.txt`); + await touch(otherUserId, `${otherAnchor}/theirs.txt`); + + await vi.waitFor( + () => expect(worker.recorded.length).toBeGreaterThanOrEqual(2), + { timeout: EVENTS_COALESCE_WINDOW_MS * 20, interval: 25 }, + ); + + const forSub = (subId: unknown) => + worker.recorded.find((call) => call.subId === subId); + + expect(forSub(mine.body.subId)).toMatchObject({ + handlerName: 'ingestUpload', + appUid, + context: JSON.stringify({ url: 'https://mine.example' }), + }); + expect(forSub(theirs.body.subId)).toMatchObject({ + handlerName: 'ingestUpload', + appUid, + context: JSON.stringify({ url: 'https://theirs.example' }), + }); + }); +}); diff --git a/src/backend/services/events/registry.ts b/src/backend/services/events/registry.ts index 3f2263410..aba9c6ec0 100644 --- a/src/backend/services/events/registry.ts +++ b/src/backend/services/events/registry.ts @@ -76,7 +76,10 @@ export type GapReason = | 'matched_subscription_limit' | 'filter_evaluation_limit' | 'delivery_rate_limit' - | 'backlog_overflow'; + | 'backlog_overflow' + // A suspension holds what it is owed only for as long as the suspension is + // plausibly recoverable; past that the backlog goes and this stands in. + | 'suspended_backlog_expired'; /** * `gap` says an event existed and was not delivered. It rides the delivery diff --git a/src/backend/services/events/suspension.test.ts b/src/backend/services/events/suspension.test.ts new file mode 100644 index 000000000..32e9dec3e --- /dev/null +++ b/src/backend/services/events/suspension.test.ts @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { describe, expect, it } from 'vitest'; +import { + EVENTS_NO_CREDIT_BACKLOG_TTL_MS, + EVENTS_SUSPENDED_BACKLOG_TTL_MS, + EVENTS_SUSPENDED_PENDING_CAP, +} from '../../controllers/events/limits.js'; +import { SUSPENDED_REASONS } from '../../stores/events/DurableSubscriptionStore.js'; +import { + backlogPolicyFor, + isMetered, + isResumable, + suspendedFor, +} from './suspension.js'; + +describe('the backlog a suspension holds', () => { + it('holds a reduced cap for every reason that can come back', () => { + for (const reason of ['handler_not_found', 'failures', 'no_credit'] as const) { + const policy = backlogPolicyFor(reason); + expect(policy.cap).toBe(EVENTS_SUSPENDED_PENDING_CAP); + expect(policy.ttlMs).toBeGreaterThan(0); + expect(policy.resumable).toBe(true); + } + }); + + it('gives a lapsed balance a shorter window than a bad deploy', () => { + expect(backlogPolicyFor('handler_not_found').ttlMs).toBe( + EVENTS_SUSPENDED_BACKLOG_TTL_MS, + ); + expect(backlogPolicyFor('failures').ttlMs).toBe( + EVENTS_SUSPENDED_BACKLOG_TTL_MS, + ); + expect(backlogPolicyFor('no_credit').ttlMs).toBe( + EVENTS_NO_CREDIT_BACKLOG_TTL_MS, + ); + expect(backlogPolicyFor('no_credit').ttlMs).toBeLessThan( + backlogPolicyFor('failures').ttlMs, + ); + }); + + it('keeps nothing at all for a withdrawn grant', () => { + // The backlog names paths its holder has just lost the right to see, + // and the suspension by design never lifts. + expect(backlogPolicyFor('permission_revoked')).toMatchObject({ + cap: 0, + ttlMs: 0, + resumable: false, + }); + }); + + it('has a policy for every reason a row can carry', () => { + for (const reason of SUSPENDED_REASONS) + expect(backlogPolicyFor(reason)).toBeDefined(); + }); +}); + +describe('what a suspension state answers', () => { + it('resumes everything but a withdrawn grant', () => { + expect(isResumable('handler_not_found')).toBe(true); + expect(isResumable('failures')).toBe(true); + expect(isResumable('no_credit')).toBe(true); + expect(isResumable('permission_revoked')).toBe(false); + }); + + it('stops metering while a row is out of service', () => { + expect(isMetered({ suspendedAt: null })).toBe(true); + expect(isMetered({ suspendedAt: 1_700_000_000 })).toBe(false); + }); + + it('matches a row against the reason a resume would lift', () => { + const row = { suspendedAt: 1, suspendedReason: 'handler_not_found' }; + expect(suspendedFor(row, 'handler_not_found')).toBe(true); + expect(suspendedFor(row, 'no_credit')).toBe(false); + expect( + suspendedFor( + { suspendedAt: null, suspendedReason: null }, + 'handler_not_found', + ), + ).toBe(false); + }); +}); diff --git a/src/backend/services/events/suspension.ts b/src/backend/services/events/suspension.ts new file mode 100644 index 000000000..953da3ea7 --- /dev/null +++ b/src/backend/services/events/suspension.ts @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { + EVENTS_NO_CREDIT_BACKLOG_TTL_MS, + EVENTS_SUSPENDED_BACKLOG_TTL_MS, + EVENTS_SUSPENDED_PENDING_CAP, +} from '../../controllers/events/limits.js'; +import type { DurableSubscription } from '../../stores/events/types.js'; +import type { SuspendedReason } from '../../stores/events/DurableSubscriptionStore.js'; + +/** + * Suspended-versus-active on a durable subscription, and what each reason does + * to what the subscription is owed. + * + * Suspension is a state, not a deletion, so a bad deploy or a lapsed card is + * recoverable: the row keeps its identity, stops being delivered to, stops + * being metered, and comes out of every watched set until it resumes. + * + * The backlog policy is per reason because "stops metering" plus "holds + * backlog" is a free memory hold — removing one widely-subscribed handler would + * otherwise convert every dependent into a full unbilled backlog held forever. + * A revoked grant is the one that purges: its backlog names paths its holder + * has just lost the right to see, and keeping them for a resume that by design + * never comes turns a revocation into a delayed disclosure. + */ + +/** What a suspension does to the deliveries the subscription is still owed. */ +export interface BacklogPolicy { + /** Deliveries kept; the rest are shed with a gap marker in their place. */ + cap: number; + /** How long the kept ones survive, or `0` to drop them now. */ + ttlMs: number; + /** Whether the reason can be lifted at all. */ + resumable: boolean; +} + +const HELD: BacklogPolicy = { + cap: EVENTS_SUSPENDED_PENDING_CAP, + ttlMs: EVENTS_SUSPENDED_BACKLOG_TTL_MS, + resumable: true, +}; + +export const BACKLOG_POLICY: Record = { + handler_not_found: HELD, + failures: HELD, + // The resume condition is a top-up, so the window is shorter. + no_credit: { + cap: EVENTS_SUSPENDED_PENDING_CAP, + ttlMs: EVENTS_NO_CREDIT_BACKLOG_TTL_MS, + resumable: true, + }, + permission_revoked: { cap: 0, ttlMs: 0, resumable: false }, +}; + +export const backlogPolicyFor = (reason: SuspendedReason): BacklogPolicy => + BACKLOG_POLICY[reason]; + +/** Whether a reason ever lifts. `permission_revoked` never does. */ +export const isResumable = (reason: SuspendedReason): boolean => + BACKLOG_POLICY[reason].resumable; + +/** + * Whether a subscription accrues metering lines. The seam the metering work + * reads: a suspended subscription is not delivering, so it is not billed for + * standing there. + */ +export const isMetered = ( + row: Pick, +): boolean => row.suspendedAt === null; + +/** Whether a suspended row is in the state a given resume would lift. */ +export const suspendedFor = ( + row: Pick, + reason: SuspendedReason, +): boolean => row.suspendedAt !== null && row.suspendedReason === reason; diff --git a/src/backend/stores/events/DurableSubscriptionStore.integration.test.ts b/src/backend/stores/events/DurableSubscriptionStore.integration.test.ts index a71e650c4..10641a667 100644 --- a/src/backend/stores/events/DurableSubscriptionStore.integration.test.ts +++ b/src/backend/stores/events/DurableSubscriptionStore.integration.test.ts @@ -58,7 +58,11 @@ const input = ( match: null, op: null, delivery: 'broadcast', - targets: ['socket', 'worker'], + // Null `appUid` is the default here (a session/account row), and a + // worker target needs an app to run one for — see "one events worker per + // app" below. Tests that want an app-scoped row with a worker target + // override both. + targets: ['socket'], handlerName: null, context: null, permission: 'list', @@ -103,7 +107,12 @@ describe('creating a subscription', () => { it('round-trips the row and names it after the app that made it', async () => { const appUid = `app-${uuidv4()}`; const { row } = await durable().create( - input({ appUid, handlerName: 'onWrite', match: '*.txt' }), + input({ + appUid, + handlerName: 'onWrite', + match: '*.txt', + targets: ['socket', 'worker'], + }), ); expect(row.subId.startsWith(`${appUid}#`)).toBe(true); @@ -205,6 +214,25 @@ describe('validation at the row write', () => { expect(row.targets).toEqual(['socket', 'push']); }); + it('refuses a `worker` target on a row with no app', async () => { + // Exactly one events worker per app: a row nobody's app made has + // nothing for that target to invoke. + await expect( + durable().create(input({ appUid: null, targets: ['socket', 'worker'] })), + ).rejects.toSatisfy(codeOf('invalid_targets')); + await expect(durable().countForHolder(userId)).resolves.toBe(0); + }); + + it('allows a `worker` target once the row has an app', async () => { + const { row } = await durable().create( + input({ + appUid: `app-${uuidv4()}`, + targets: ['socket', 'worker'], + }), + ); + expect(row.targets).toEqual(['socket', 'worker']); + }); + it('refuses a context past the hard cap', async () => { await expect( durable().create(input({ context: 'x'.repeat(4097) })), diff --git a/src/backend/stores/events/DurableSubscriptionStore.ts b/src/backend/stores/events/DurableSubscriptionStore.ts index aa8034b36..a243135b0 100644 --- a/src/backend/stores/events/DurableSubscriptionStore.ts +++ b/src/backend/stores/events/DurableSubscriptionStore.ts @@ -65,6 +65,12 @@ export const DURABLE_CONTEXT_MAX_BYTES = 4096; export const DURABLE_LIST_DEFAULT_LIMIT = 50; export const DURABLE_LIST_LIMIT_CAP = 200; +/** + * Rows one handler-lifecycle pass takes. A removal suspends its dependents in + * batches so a widely-used name cannot make one call hold the whole set. + */ +export const HANDLER_SETTLE_BATCH = 500; + const TABLE = 'event_subscriptions'; // -- Wire shapes ------------------------------------------------------ @@ -96,8 +102,24 @@ export interface DurableListOptions { includeTotal?: boolean; } -/** Why a row is out of service. Never auto-resumes for `permission_revoked`. */ -export type SuspendedReason = 'permission_revoked'; +/** + * Why a row is out of service. + * + * All four are states rather than deletions, so a bad deploy or a lapsed card + * is recoverable. Only `permission_revoked` is terminal: consent to watch is + * re-established by subscribing again, never by re-granting. + */ +export const SUSPENDED_REASONS = [ + 'handler_not_found', + 'failures', + 'no_credit', + 'permission_revoked', +] as const; + +export type SuspendedReason = (typeof SUSPENDED_REASONS)[number]; + +export const isSuspendedReason = (value: unknown): value is SuspendedReason => + SUSPENDED_REASONS.includes(value as SuspendedReason); /** Where a row is moving to when its anchor is deleted under it. */ export interface ReanchorInput { @@ -132,6 +154,13 @@ const pushOnSingle = (): HttpError => { legacyCode: 'invalid_targets' }, ); +const workerNeedsApp = (): HttpError => + new HttpError( + 400, + 'A subscription with no app has no events worker to target', + { legacyCode: 'invalid_targets' }, + ); + const quotaReached = (): HttpError => new HttpError( 429, @@ -223,7 +252,11 @@ export class DurableSubscriptionStore extends PuterStore { async create( input: DurableSubscriptionInput, ): Promise<{ row: DurableSubscription; bump: GenerationBump }> { - const targets = this.#assertTargets(input.delivery, input.targets); + const targets = this.#assertTargets( + input.delivery, + input.targets, + input.appUid, + ); this.#assertContext(input.context); const held = await this.countForHolder(input.holderUserId); @@ -333,6 +366,44 @@ export class DurableSubscriptionStore extends PuterStore { return { suspended, bumps }; } + /** + * Put suspended rows back in service: clear the state, put each back in + * this region's cache, and bump so every other region rebuilds. The inverse + * of `suspend`, and the only way back for the three reasons that resume. + */ + async resume( + rows: readonly DurableSubscription[], + ): Promise { + if (rows.length === 0) return []; + + await this.clients.db.write( + `UPDATE \`${TABLE}\` SET \`suspended_at\` = NULL, ` + + '`suspended_reason` = NULL WHERE `sub_id` IN ' + + `(${rows.map(() => '?').join(', ')})`, + rows.map((row) => row.subId), + ); + + // Cached per owner: the cache keys one hash per owner and takes the + // owner from the first row it is given. + const byOwner = new Map(); + for (const row of rows) { + const live: DurableSubscription = { + ...row, + suspendedAt: null, + suspendedReason: null, + }; + const held = byOwner.get(row.ownerUserId); + if (held) held.push(live); + else byOwner.set(row.ownerUserId, [live]); + } + for (const [, owned] of byOwner) + await this.stores.eventSubscription.cacheDurable(owned); + + return Promise.all( + [...byOwner.keys()].map((owner) => this.#bump(owner)), + ); + } + /** * Move one row onto a different anchor, keeping its identity. The cache * entry moves with it — including across owners, which is a different @@ -518,6 +589,62 @@ export class DurableSubscriptionStore extends PuterStore { return rows.map(toRow); } + /** + * The rows bound to one of an app's handler names. What a removal has to + * suspend, and — asking for the suspended half — what a republish resumes. + * + * Bounded by the per-account quota times nothing: a widely-used handler can + * carry more rows than one read should return, so this is a page and the + * caller walks it until it comes back short. + */ + async listByHandler( + appUid: string, + handlerName: string, + options: { suspendedReason?: SuspendedReason; limit?: number } = {}, + ): Promise { + const where = ['`app_uid` = ?', '`handler_name` = ?']; + const params: unknown[] = [appUid, handlerName]; + if (options.suspendedReason === undefined) { + where.push('`suspended_at` IS NULL'); + } else { + where.push('`suspended_at` IS NOT NULL', '`suspended_reason` = ?'); + params.push(options.suspendedReason); + } + + const rows = await this.clients.db.pread( + `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` + + `WHERE ${where.join(' AND ')} ORDER BY \`id\` LIMIT ?`, + [ + ...params, + Math.max(1, Math.floor(options.limit ?? HANDLER_SETTLE_BATCH)), + ], + ); + return rows.map(toRow); + } + + /** + * One holder's rows suspended for a given reason — what a resume condition + * that belongs to the account rather than to a handler releases. + */ + async listSuspendedForHolder( + holderUserId: number, + reason: SuspendedReason, + ): Promise { + const rows = await this.clients.db.pread( + `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` + + 'WHERE `holder_user_id` = ? AND `suspended_at` IS NOT NULL ' + + `AND \`suspended_reason\` = ? AND ${this.#unexpiredClause()} ` + + 'ORDER BY `id` LIMIT ?', + [ + holderUserId, + reason, + nowSeconds(), + EVENTS_DURABLE_SUBSCRIPTIONS_PER_USER, + ], + ); + return rows.map(toRow); + } + /** * Every row a region has to be able to deliver for one owner. Read from the * primary: this is what a cold region caches, and caching a replica's "no @@ -597,13 +724,16 @@ export class DurableSubscriptionStore extends PuterStore { } /** - * The row cannot exist with transports its delivery class cannot use. Held - * here rather than only at the API, so a writer that never passes through - * one cannot leave an unsatisfiable row behind. + * The row cannot exist with transports its delivery class cannot use, or + * with a `worker` target and no app to run one for — "one events worker per + * app" means no app is no worker target, not a worker with nowhere to go. + * Held here rather than only at the API, so a writer that never passes + * through one cannot leave an unsatisfiable row behind. */ #assertTargets( delivery: DeliveryClass, targets: readonly string[], + appUid: string | null, ): SubscriptionTarget[] { if (!Array.isArray(targets) || targets.length === 0) throw invalidTargets(); @@ -611,6 +741,8 @@ export class DurableSubscriptionStore extends PuterStore { const unique = [...new Set(targets as SubscriptionTarget[])]; if (!targetsAllowedForDelivery(delivery, unique)) throw pushOnSingle(); + if (appUid === null && unique.includes('worker')) + throw workerNeedsApp(); return unique; } diff --git a/src/backend/stores/events/EventHandlerStore.integration.test.ts b/src/backend/stores/events/EventHandlerStore.integration.test.ts new file mode 100644 index 000000000..00580869d --- /dev/null +++ b/src/backend/stores/events/EventHandlerStore.integration.test.ts @@ -0,0 +1,351 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Handler CRUD against a real table. What is on the hook is the publish + * contract: same source is a no-op, different source needs the caller to say + * which one it is replacing, and the name is scoped to one app. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { v4 as uuidv4 } from 'uuid'; +import { EVENTS_HANDLER_SOURCE_MAX_BYTES } from '../../controllers/events/limits.js'; +import { isHttpError } from '../../core/http/HttpError.js'; +import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js'; +import type { IConfig } from '../../types.js'; +import { hashContent } from './EventHandlerStore.js'; + +const BOOT_TIMEOUT_MS = 120_000; + +let env: PuterTestEnv; +let appUid: string; +let otherAppUid: string; + +const handlers = () => env.server.stores.eventHandler; + +const SOURCE = 'async ({ event }) => { console.log(event.path); }'; +const OTHER_SOURCE = 'async ({ event, ctx }) => { console.log(ctx.url); }'; + +const codeOf = (code: string) => (err: unknown) => + isHttpError(err) && err.legacyCode === code; + +beforeAll(async () => { + env = await setupPuterTestEnv({ events: { enabled: true } } as IConfig); + appUid = `app-${uuidv4()}`; + otherAppUid = `app-${uuidv4()}`; +}, BOOT_TIMEOUT_MS); + +afterAll(async () => { + await env?.shutdown(); +}); + +beforeEach(async () => { + await env.server.clients.db.write('DELETE FROM `event_handlers`', []); + await env.server.clients.db.write('DELETE FROM `event_subscriptions`', []); +}); + +describe('publishing a handler', () => { + it('creates a name that was not there', async () => { + const { handler, outcome } = await handlers().publish({ + appUid, + name: 'ingestUpload', + source: SOURCE, + }); + + expect(outcome).toBe('created'); + expect(handler.sourceHash).toBe(hashContent(SOURCE)); + await expect( + handlers().getByName(appUid, 'ingestUpload'), + ).resolves.toMatchObject({ name: 'ingestUpload', source: SOURCE }); + }); + + it('is a no-op when the same source is published again', async () => { + const first = await handlers().publish({ + appUid, + name: 'ingestUpload', + source: SOURCE, + }); + const again = await handlers().publish({ + appUid, + name: 'ingestUpload', + source: SOURCE, + }); + + expect(again.outcome).toBe('unchanged'); + expect(again.handler.updatedAt).toBe(first.handler.updatedAt); + }); + + it('refuses different source from a publisher that did not name the base', async () => { + await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE }); + + await expect( + handlers().publish({ + appUid, + name: 'ingestUpload', + source: OTHER_SOURCE, + }), + ).rejects.toSatisfy(codeOf('events_handler_conflict')); + await expect( + handlers().getByName(appUid, 'ingestUpload'), + ).resolves.toMatchObject({ source: SOURCE }); + }); + + it('takes the update from a publisher whose base is what is published', async () => { + const first = await handlers().publish({ + appUid, + name: 'ingestUpload', + source: SOURCE, + }); + + const updated = await handlers().publish({ + appUid, + name: 'ingestUpload', + source: OTHER_SOURCE, + ifHash: first.handler.sourceHash, + }); + + expect(updated.outcome).toBe('updated'); + expect(updated.handler.sourceHash).toBe(hashContent(OTHER_SOURCE)); + }); + + it('refuses the second of two build steps that both branched from the same base', async () => { + const base = await handlers().publish({ + appUid, + name: 'ingestUpload', + source: SOURCE, + }); + + await handlers().publish({ + appUid, + name: 'ingestUpload', + source: OTHER_SOURCE, + ifHash: base.handler.sourceHash, + }); + + await expect( + handlers().publish({ + appUid, + name: 'ingestUpload', + source: 'async () => { /* a third build */ }', + ifHash: base.handler.sourceHash, + }), + ).rejects.toSatisfy(codeOf('events_handler_conflict')); + }); + + it('takes the name outright for a publisher that asked to replace', async () => { + await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE }); + + const replaced = await handlers().publish({ + appUid, + name: 'ingestUpload', + source: OTHER_SOURCE, + replace: true, + }); + + expect(replaced.outcome).toBe('updated'); + expect(replaced.handler.source).toBe(OTHER_SOURCE); + }); + + it('refuses an update whose base is not published at all', async () => { + await expect( + handlers().publish({ + appUid, + name: 'ingestUpload', + source: SOURCE, + ifHash: hashContent(OTHER_SOURCE), + }), + ).rejects.toSatisfy(codeOf('events_handler_conflict')); + }); + + it('lets exactly one of two racing different-source updates win', async () => { + const base = await handlers().publish({ + appUid, + name: 'ingestUpload', + source: SOURCE, + }); + + const results = await Promise.allSettled([ + handlers().publish({ + appUid, + name: 'ingestUpload', + source: OTHER_SOURCE, + ifHash: base.handler.sourceHash, + }), + handlers().publish({ + appUid, + name: 'ingestUpload', + source: 'async () => { /* the other racer */ }', + ifHash: base.handler.sourceHash, + }), + ]); + + const fulfilled = results.filter((r) => r.status === 'fulfilled'); + const rejected = results.filter((r) => r.status === 'rejected'); + expect(fulfilled).toHaveLength(1); + expect(rejected).toHaveLength(1); + expect((rejected[0] as PromiseRejectedResult).reason).toSatisfy( + codeOf('events_handler_conflict'), + ); + + // The row reflects whichever one actually won — never a mix of the + // two, and never silently both. + const stored = await handlers().getByName(appUid, 'ingestUpload'); + expect([OTHER_SOURCE, 'async () => { /* the other racer */ }']).toContain( + stored!.source, + ); + }); + + it('never lets a create-vs-create race leak a raw database error', async () => { + const results = await Promise.allSettled([ + handlers().publish({ + appUid, + name: 'brandNew', + source: SOURCE, + }), + handlers().publish({ + appUid, + name: 'brandNew', + source: OTHER_SOURCE, + }), + ]); + + const fulfilled = results.filter((r) => r.status === 'fulfilled'); + const rejected = results.filter( + (r): r is PromiseRejectedResult => r.status === 'rejected', + ); + // Exactly one side creates the name; racing straight into a UNIQUE + // index must surface as the same stable conflict a sequential caller + // gets, never a raw driver error escaping the store. + expect(fulfilled).toHaveLength(1); + expect(rejected).toHaveLength(1); + expect(rejected[0].reason).toSatisfy(codeOf('events_handler_conflict')); + + const stored = await handlers().getByName(appUid, 'brandNew'); + expect([SOURCE, OTHER_SOURCE]).toContain(stored!.source); + }); + + it('scopes the name to one app', async () => { + await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE }); + const other = await handlers().publish({ + appUid: otherAppUid, + name: 'ingestUpload', + source: OTHER_SOURCE, + }); + + expect(other.outcome).toBe('created'); + await expect( + handlers().getByName(appUid, 'ingestUpload'), + ).resolves.toMatchObject({ source: SOURCE }); + }); + + it('refuses a name that is not an addressable identifier', async () => { + for (const name of ['', ' leading', 'has space', '-dash', 'x'.repeat(129)]) { + await expect( + handlers().publish({ appUid, name, source: SOURCE }), + ).rejects.toSatisfy(codeOf('events_handler_name_invalid')); + } + }); + + it('refuses empty source and source past the cap', async () => { + await expect( + handlers().publish({ appUid, name: 'empty', source: ' ' }), + ).rejects.toSatisfy(codeOf('events_handler_source_invalid')); + + await expect( + handlers().publish({ + appUid, + name: 'huge', + source: 'x'.repeat(EVENTS_HANDLER_SOURCE_MAX_BYTES + 1), + }), + ).rejects.toSatisfy(codeOf('events_handler_too_large')); + }); +}); + +describe('listing handlers', () => { + it('reports names, hashes and dependents, never source', async () => { + await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE }); + await handlers().publish({ + appUid, + name: 'indexDocument', + source: OTHER_SOURCE, + }); + + const listed = await handlers().listForApp(appUid); + + expect(listed.map((row) => row.name)).toEqual([ + 'indexDocument', + 'ingestUpload', + ]); + expect(listed[1]).toMatchObject({ + name: 'ingestUpload', + hash: hashContent(SOURCE), + subscriptions: 0, + }); + expect( + listed.every((row) => !('source' in row)), + ).toBe(true); + }); + + it('counts the subscriptions each name is carrying', async () => { + await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE }); + for (const suffix of ['a', 'b']) { + await env.server.clients.db.write( + 'INSERT INTO `event_subscriptions` (`sub_id`, `token`, `owner_user_id`, ' + + '`holder_user_id`, `app_uid`, `subject`, `anchor_uid`, `anchor_path`, ' + + '`delivery`, `handler_name`, `targets`, `permission`, `created_at`) ' + + 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + [ + `${appUid}#${suffix}`, + 'f#anchor', + 1, + 1, + appUid, + 'fs:/x', + 'anchor', + '/x', + 'single', + 'ingestUpload', + JSON.stringify(['worker']), + 'list', + 0, + ], + ); + } + + const [listed] = await handlers().listForApp(appUid); + expect(listed.subscriptions).toBe(2); + }); +}); + +describe('removing a handler', () => { + it('drops the row and reports what it was', async () => { + await handlers().publish({ appUid, name: 'ingestUpload', source: SOURCE }); + + await expect( + handlers().remove(appUid, 'ingestUpload'), + ).resolves.toMatchObject({ name: 'ingestUpload' }); + await expect( + handlers().getByName(appUid, 'ingestUpload'), + ).resolves.toBeNull(); + }); + + it('answers null for a name the app never published', async () => { + await expect(handlers().remove(appUid, 'nothing')).resolves.toBeNull(); + }); +}); diff --git a/src/backend/stores/events/EventHandlerStore.ts b/src/backend/stores/events/EventHandlerStore.ts new file mode 100644 index 000000000..deb105c2b --- /dev/null +++ b/src/backend/stores/events/EventHandlerStore.ts @@ -0,0 +1,380 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { createHash } from 'node:crypto'; +import { + EVENTS_HANDLERS_PER_APP, + EVENTS_HANDLER_SOURCE_MAX_BYTES, +} from '../../controllers/events/limits.js'; +import { HttpError } from '../../core/http/HttpError.js'; +import { isUniqueViolation } from '../../util/dbError.js'; +import { PuterStore } from '../types.js'; + +/** + * Named handlers an app has deployed. + * + * A handler name is a label for a piece of code, not an event: nothing triggers + * by name, and a row here runs only when a subscription bound to that name has + * a delivery. The name is the identity — it survives source changes, and it is + * what subscriptions bind to; the hash is only a change detector and an + * idempotency key. + * + * Two writers race on every build step, so `publish` is optimistic rather than + * last-write-wins: a caller says which source it believes is published + * (`ifHash`), and a publish whose base has moved under it is refused rather + * than silently picking a winner. + */ + +const TABLE = 'event_handlers'; +const SUBSCRIPTION_TABLE = 'event_subscriptions'; + +/** Longest a handler name may be, matching the column that holds it. */ +export const HANDLER_NAME_MAX_LENGTH = 128; + +/** Names are addressable identifiers, not free text. */ +const NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]*$/; + +// -- Shapes ----------------------------------------------------------- + +/** A row in `event_handlers`, as the rest of the system sees it. */ +export interface EventHandler { + appUid: string; + name: string; + source: string; + sourceHash: string; + createdAt: number; + updatedAt: number; +} + +/** One handler as `list` reports it. Never carries source. */ +export interface EventHandlerSummary { + name: string; + hash: string; + updatedAt: number; + /** Subscriptions currently bound to this name, suspended ones included. */ + subscriptions: number; +} + +export interface PublishHandlerInput { + appUid: string; + name: string; + source: string; + /** + * The hash the caller believes is published. Absent means "create, or + * accept that it is already exactly this" — anything else is a conflict. + */ + ifHash?: string | null; + /** Take the name whatever is published under it. */ + replace?: boolean; +} + +/** What one publish did, which is what the caller reports back. */ +export type PublishOutcome = 'created' | 'updated' | 'unchanged'; + +export interface PublishResult { + handler: EventHandler; + outcome: PublishOutcome; +} + +// -- Errors ----------------------------------------------------------- + +const invalidName = (): HttpError => + new HttpError( + 400, + 'A handler name must start alphanumeric and may contain letters, ' + + `digits, and \`_ . : -\`, up to ${HANDLER_NAME_MAX_LENGTH} characters`, + { legacyCode: 'events_handler_name_invalid' }, + ); + +const sourceTooLarge = (): HttpError => + new HttpError( + 413, + `A handler may not exceed ${EVENTS_HANDLER_SOURCE_MAX_BYTES} bytes`, + { legacyCode: 'events_handler_too_large' }, + ); + +const invalidSource = (): HttpError => + new HttpError(400, 'A handler must be a non-empty source string', { + legacyCode: 'events_handler_source_invalid', + }); + +const tooManyHandlers = (): HttpError => + new HttpError( + 429, + `An app may publish ${EVENTS_HANDLERS_PER_APP} handlers`, + { legacyCode: 'events_handler_limit' }, + ); + +/** + * Two build steps racing. Refused rather than resolved: whichever won would be + * running the other's users' subscriptions, and neither asked for that. + */ +const publishConflict = (name: string): HttpError => + new HttpError( + 409, + `\`${name}\` has different source published; pass \`replace\` to take it`, + { legacyCode: 'events_handler_conflict' }, + ); + +// -- Row mapping ------------------------------------------------------ + +const nowSeconds = (): number => Math.floor(Date.now() / 1000); + +const toRow = (row: Record): EventHandler => ({ + appUid: String(row.app_uid), + name: String(row.name), + source: String(row.source), + sourceHash: String(row.source_hash), + createdAt: Number(row.created_at) || 0, + updatedAt: Number(row.updated_at) || 0, +}); + +const SELECT_COLUMNS = + '`app_uid`, `name`, `source`, `source_hash`, `created_at`, `updated_at`'; + +/** + * Content hash of a stored blob. The handler change detector, what an inline + * subscribe body is checked against, and what a subscription listing reports in + * place of its context values. + */ +export const hashContent = (content: string): string => + createHash('sha256').update(content, 'utf8').digest('hex'); + +export const isValidHandlerName = (name: string): boolean => + name.length > 0 && + name.length <= HANDLER_NAME_MAX_LENGTH && + NAME_PATTERN.test(name); + +export class EventHandlerStore extends PuterStore { + // -- Writes ------------------------------------------------------ + + /** + * Create or update one handler. + * + * Reads the primary: a publish decided against a lagging replica would + * either resurrect source the previous call replaced, or report a conflict + * with a row that no longer exists. + */ + async publish(input: PublishHandlerInput): Promise { + const name = this.#assertName(input.name); + const source = this.#assertSource(input.source); + const sourceHash = hashContent(source); + + const existing = await this.getByName(input.appUid, name); + if (!existing) { + // `ifHash` naming a source that is not there is the same race from + // the other side: something removed the row this publish was + // updating, and re-creating it silently would undo that. + if (input.ifHash && !input.replace) throw publishConflict(name); + return { + handler: await this.#insert( + input.appUid, + name, + source, + sourceHash, + ), + outcome: 'created', + }; + } + + // Idempotent whatever the caller believed: the published source is + // already the one being asked for. + if (existing.sourceHash === sourceHash) + return { handler: existing, outcome: 'unchanged' }; + + const basedOnPublished = + input.ifHash !== undefined && + input.ifHash !== null && + input.ifHash === existing.sourceHash; + if (!input.replace && !basedOnPublished) throw publishConflict(name); + + return { + handler: await this.#update(existing, source, sourceHash), + outcome: 'updated', + }; + } + + /** Drop one handler. Null when the app had nothing published by that name. */ + async remove(appUid: string, name: string): Promise { + const existing = await this.getByName(appUid, name); + if (!existing) return null; + await this.clients.db.write( + `DELETE FROM \`${TABLE}\` WHERE \`app_uid\` = ? AND \`name\` = ?`, + [appUid, name], + ); + return existing; + } + + // -- Reads ------------------------------------------------------- + + /** + * One handler by name. Primary: this answers both the binding check a + * subscribe runs and the base a publish is compared against, and a replica + * behind by a moment would report a handler that was just published as + * absent. + */ + async getByName( + appUid: string, + name: string, + ): Promise { + const rows = await this.clients.db.pread( + `SELECT ${SELECT_COLUMNS} FROM \`${TABLE}\` ` + + 'WHERE `app_uid` = ? AND `name` = ?', + [appUid, name], + ); + return rows.length > 0 ? toRow(rows[0]) : null; + } + + /** + * What an app has published, with how many subscriptions each name is + * carrying. Bounded by the per-app cap, so the whole set is one page. + * + * The counts come from one grouped read rather than a join, because the + * subscription side has to count names an app never published — a handler + * removed while subscriptions still pointed at it leaves exactly that. + */ + async listForApp(appUid: string): Promise { + const rows = await this.clients.db.read( + 'SELECT `name`, `source_hash`, `updated_at` ' + + `FROM \`${TABLE}\` WHERE \`app_uid\` = ? ` + + 'ORDER BY `name` LIMIT ?', + [appUid, EVENTS_HANDLERS_PER_APP], + ); + + const counts = await this.countSubscriptionsByHandler(appUid); + return rows.map((row) => ({ + name: String(row.name), + hash: String(row.source_hash), + updatedAt: Number(row.updated_at) || 0, + subscriptions: counts.get(String(row.name)) ?? 0, + })); + } + + async countForApp(appUid: string): Promise { + const [row] = await this.clients.db.pread( + `SELECT COUNT(*) AS \`total\` FROM \`${TABLE}\` WHERE \`app_uid\` = ?`, + [appUid], + ); + return Number(row?.total ?? 0); + } + + /** + * How many subscriptions each of an app's handler names is carrying, + * suspended rows included — a suspended subscription is still a dependent, + * and it is the reason a removal is not a delete. + */ + async countSubscriptionsByHandler( + appUid: string, + ): Promise> { + const rows = await this.clients.db.read( + 'SELECT `handler_name`, COUNT(*) AS `total` ' + + `FROM \`${SUBSCRIPTION_TABLE}\` ` + + 'WHERE `app_uid` = ? AND `handler_name` IS NOT NULL ' + + 'GROUP BY `handler_name`', + [appUid], + ); + return new Map( + rows.map((row) => [ + String(row.handler_name), + Number(row.total) || 0, + ]), + ); + } + + // -- Internals --------------------------------------------------- + + async #insert( + appUid: string, + name: string, + source: string, + sourceHash: string, + ): Promise { + if ((await this.countForApp(appUid)) >= EVENTS_HANDLERS_PER_APP) + throw tooManyHandlers(); + + const at = nowSeconds(); + try { + await this.clients.db.insert(TABLE, { + app_uid: appUid, + name, + source, + source_hash: sourceHash, + created_at: at, + updated_at: at, + }); + } catch (err) { + // Lost a create-vs-create race: the unique index is what actually + // arbitrated it, and the loser hits it here rather than at the + // read above. Same stable code a sequential caller gets — never + // the raw driver error. + if (isUniqueViolation(err)) throw publishConflict(name); + throw err; + } + return { + appUid, + name, + source, + sourceHash, + createdAt: at, + updatedAt: at, + }; + } + + async #update( + existing: EventHandler, + source: string, + sourceHash: string, + ): Promise { + const at = nowSeconds(); + // The `source_hash` predicate is what makes the check-then-write a + // compare-and-set: a publish that raced past the read above updates + // nothing here, and its caller is told. + const updated = await this.clients.db.write( + `UPDATE \`${TABLE}\` SET \`source\` = ?, \`source_hash\` = ?, ` + + '`updated_at` = ? WHERE `app_uid` = ? AND `name` = ? ' + + 'AND `source_hash` = ?', + [ + source, + sourceHash, + at, + existing.appUid, + existing.name, + existing.sourceHash, + ], + ); + if (updated?.anyRowsAffected === false) + throw publishConflict(existing.name); + + return { ...existing, source, sourceHash, updatedAt: at }; + } + + #assertName(name: unknown): string { + if (typeof name !== 'string' || !isValidHandlerName(name)) + throw invalidName(); + return name; + } + + #assertSource(source: unknown): string { + if (typeof source !== 'string' || source.trim().length === 0) + throw invalidSource(); + if (Buffer.byteLength(source, 'utf8') > EVENTS_HANDLER_SOURCE_MAX_BYTES) + throw sourceTooLarge(); + return source; + } +} diff --git a/src/backend/stores/events/PendingDeliveryStore.ts b/src/backend/stores/events/PendingDeliveryStore.ts index beae7eae5..e49e5201e 100644 --- a/src/backend/stores/events/PendingDeliveryStore.ts +++ b/src/backend/stores/events/PendingDeliveryStore.ts @@ -71,6 +71,7 @@ import { PuterStore } from '../types.js'; const entriesKey = (subId: string): string => `ev:q:{${subId}}`; const pendingKey = (subId: string): string => `ev:qp:{${subId}}`; const leaseKey = (subId: string): string => `ev:ql:{${subId}}`; +const holdKey = (subId: string): string => `ev:qt:{${subId}}`; const INDEX_KEY = 'ev:qx'; const COUNTER_KEY = 'ev:qc'; @@ -198,11 +199,14 @@ const parseEntry = (raw: string | null): StoredEntry | null => { } }; -const gapMarker = (subject: string): GapMarker => ({ +const gapMarker = ( + subject: string, + reason: GapMarker['reason'] = 'backlog_overflow', +): GapMarker => ({ id: randomUUID(), subject, op: 'gap', - reason: 'backlog_overflow', + reason, ts: Date.now(), }); @@ -362,12 +366,71 @@ export class PendingDeliveryStore extends PuterStore { entriesKey(subId), pendingKey(subId), leaseKey(subId), + holdKey(subId), ); await this.clients.redis.zrem(INDEX_KEY, subId); if (held > 0) await this.clients.redis.decrby(COUNTER_KEY, held); return held; } + /** + * Take a subscription's backlog down to `cap` and put a deadline on what is + * left. What a suspension does to what it is owed: a suspended subscription + * stops being metered, so holding its full backlog is an unbilled memory + * hold, and holding it forever is one that never comes back. + * + * The deadline is enforced by the sweeper rather than by a key TTL — the + * entries have to be dropped with a marker in their place, and an expiring + * key would take them silently. + */ + async hold( + subId: string, + cap: number, + ttlMs: number, + ): Promise { + await this.clients.redis.set(holdKey(subId), Date.now() + ttlMs); + + const held = await this.depth(subId); + const over = held - Math.max(0, Math.floor(cap)); + if (over <= 0) return null; + // One more than the overflow, because the marker that replaces them + // takes a place of its own. + return this.#shedOldest(subId, over + 1, 'subscription'); + } + + /** + * Drop a held backlog whose deadline has passed, leaving a gap marker so + * its subscription learns there were events rather than reading the silence + * as "nothing happened". Returns how many went, or 0 while the hold + * stands. + */ + async expireHold(subId: string): Promise { + const raw = await this.clients.redis.get(holdKey(subId)); + if (raw === null) return 0; + const expiresAt = Number(raw); + if (!Number.isFinite(expiresAt) || expiresAt > Date.now()) return 0; + + const [oldest] = await this.clients.redis.zrange( + pendingKey(subId), + 0, + 0, + ); + const subject = oldest ? await this.#subjectOf(subId, oldest) : ''; + const dropped = await this.purge(subId); + if (dropped === 0) return 0; + + await this.#append( + subId, + gapMarker(subject, 'suspended_backlog_expired'), + ); + return dropped; + } + + /** Lift a hold, for a subscription that is back in service. */ + async releaseHold(subId: string): Promise { + await this.clients.redis.del(holdKey(subId)); + } + // -- Reads ------------------------------------------------------- /** diff --git a/src/backend/stores/index.ts b/src/backend/stores/index.ts index 8e1cef9c1..ce0149e47 100644 --- a/src/backend/stores/index.ts +++ b/src/backend/stores/index.ts @@ -22,6 +22,7 @@ import { AppStore } from './app/AppStore.js'; import { FSEntryStore } from './fs/FSEntryStore.js'; import { GroupStore } from './group/GroupStore.js'; import { DurableSubscriptionStore } from './events/DurableSubscriptionStore.js'; +import { EventHandlerStore } from './events/EventHandlerStore.js'; import { EventSubscriptionStore } from './events/EventSubscriptionStore.js'; import { PendingDeliveryStore } from './events/PendingDeliveryStore.js'; import { CreditHoldStore } from './metering/CreditHoldStore.js'; @@ -65,6 +66,7 @@ declare module './types.js' { userBlock: UserBlockStore; eventSubscription: EventSubscriptionStore; durableSubscription: DurableSubscriptionStore; + eventHandler: EventHandlerStore; pendingDelivery: PendingDeliveryStore; } } @@ -100,4 +102,6 @@ export const puterStores = { pendingDelivery: PendingDeliveryStore, // Writes through the Redis keyspace above, so it comes after it. durableSubscription: DurableSubscriptionStore, + // Table only, and reads the subscription table for its dependent counts. + eventHandler: EventHandlerStore, } satisfies IPuterStoreRegistry; diff --git a/src/docs/src/Events.md b/src/docs/src/Events.md index bb356f4d5..c13040d54 100644 --- a/src/docs/src/Events.md +++ b/src/docs/src/Events.md @@ -119,7 +119,7 @@ Emptying a whole store with [`puter.kv.flush()`](/KV/flush/) delivers nothing: n ### Gaps -Every per-event limit truncates the delivery rather than failing anything, and sends a **gap marker** in its place: an event with `op: 'gap'`, a `reason`, and no `uid` or `path`. A gap means something happened that you were not told the details of, so treat it as "re-read what I am watching", never as "nothing changed". +Every per-event limit truncates the delivery rather than failing anything, and sends a **gap marker** in its place: an event with `op: 'gap'`, a `reason`, and no `uid` or `path`. A gap means something happened that you were not told the details of, so treat it as "re-read what I am watching", never as "nothing changed". A persistent subscription that was suspended long enough for its held backlog to lapse gets one too, with `reason: 'suspended_backlog_expired'`. ```js await puter.events.onLocal('fs:~/Documents', async ({ event }) => { @@ -128,7 +128,7 @@ await puter.events.onLocal('fs:~/Documents', async ({ event }) => { }); ``` -## Subscriptions live with the connection +## Two kinds of subscription `onLocal()` subscriptions are **session-scoped**: nothing is stored, nothing runs while the page is closed, and the server drops them when the connection goes away. Every subscription this client makes rides one connection, which opens on the first `onLocal()` and closes when the last subscription ends. In a worker that means the subscription lasts as long as the invocation that made it, and no longer. @@ -140,11 +140,43 @@ const sub = await puter.events.onLocal('fs:~/Documents', handler, { }); ``` +[`onPersistent()`](/Events/onPersistent/) subscriptions are **stored against the account**. They keep matching with nothing open, survive every reconnect, and end only when you call [`unsubscribe()`](/Events/unsubscribe/) or their `expiresAt` passes. What runs is a *handler* your app deployed by name: + +```js +// Once, at deploy time +await puter.events.handlers.publish('ingestUpload', async ({ event, ctx }) => { + await fetch(ctx.endpoint, { method: 'POST', body: event.path }); +}, { appUid }); + +// Per user, when they opt in +await puter.events.onPersistent({ + subject: 'fs:~/inbox', + handlerName: 'ingestUpload', + context: { endpoint: 'https://example.com/ingest' }, +}); +``` + +### Handlers cannot close over anything + +A handler is deployed, not called: it is serialized with `Function.prototype.toString()` and run later, somewhere else, with nothing around it. A closed-over variable is not discouraged — it is *unrepresentable*. Every identifier a handler names has to be a parameter, something it declares itself, a standard global, or reached through `ctx`; the SDK checks that before the call and rejects with `events_handler_free_variable`, naming what it could not resolve. + +Values reach a handler through **`context`**, which is evaluated **once, at subscribe time**, serialized, and delivered to every invocation as a frozen `ctx`. It never re-evaluates: `ctx.endpoint` is whatever the value was when the subscription was created, forever, until it is created again. + +**`context` is capped at a hard 4 KB.** These are database rows read on every delivery, and `context` is the one field a developer controls the size of — over the cap the call fails with `events_context_too_large`, client-side, before the request. It is stored in plaintext and read only on the delivery path: [`list()`](/Events/list/) returns its **key names and a content hash**, never its values. For anything larger, store it in a file and put the path in `context`; a wider column is not the upgrade path. + +See [`puter.events.handlers`](/Events/handlers/) for the deploy side — publishing, replacing, and what removing a name does to the subscriptions bound to it. + +A persistent subscription can also stop without you unsubscribing: its handler was removed, its holder ran out of credit, or the share it was made under was withdrawn. It is then *suspended* rather than deleted, and [`list()`](/Events/list/) reports `suspendedAt` and `suspendedReason`. Everything but a withdrawn grant can resume. + ## Limits -Subscriptions per connection, subscribe calls per minute, and how much one event may fan out are all capped — see [Rate Limits and Quotas](/rate-limits-and-quotas/). Deliveries are coalesced over 250 ms per subject, so a multipart upload or a save loop arrives as one event rather than one per write. +Subscriptions per connection, persistent subscriptions per account, published handlers per app, subscribe calls per minute, and how much one event may fan out are all capped — see [Rate Limits and Quotas](/rate-limits-and-quotas/). Deliveries are coalesced over 250 ms per subject, so a multipart upload or a save loop arrives as one event rather than one per write. ## Functions - **[`puter.events.onLocal()`](/Events/onLocal/)** - Subscribe to a subject for as long as this client is connected -- **[`subscription.off()`](/Events/off/)** - End a subscription +- **[`subscription.off()`](/Events/off/)** - End a session subscription +- **[`puter.events.onPersistent()`](/Events/onPersistent/)** - Subscribe with a subscription that keeps running when your app is closed +- **[`puter.events.list()`](/Events/list/)** - List the persistent subscriptions this caller holds +- **[`puter.events.unsubscribe()`](/Events/unsubscribe/)** - End a persistent subscription +- **[`puter.events.handlers`](/Events/handlers/)** - Publish, list and remove the named handlers a persistent subscription runs diff --git a/src/docs/src/Events/handlers.md b/src/docs/src/Events/handlers.md new file mode 100644 index 000000000..2309a9481 --- /dev/null +++ b/src/docs/src/Events/handlers.md @@ -0,0 +1,203 @@ +--- +title: puter.events.handlers +description: Publish, list and remove the named handlers a persistent subscription runs. +platforms: [websites, apps, nodejs, workers] +--- + +
The Events API is in beta. Event shapes, limits, and behavior may change between releases.
+ +A **handler** is a function your app deploys once, under a name, that persistent subscriptions bind to. A name is a label for deployed code, not an event: nothing triggers by name, and a handler runs only when a subscription bound to it has a delivery. + +Publishing is a **developer** operation. An app token publishes into its own app; an account session has to name an app it owns with `appUid`. Either way the account must own the app. + +```js +await puter.events.handlers.publish('ingestUpload', async ({ event, ctx }) => { + await fetch(ctx.endpoint, { method: 'POST', body: event.path }); +}, { appUid }); + +await puter.events.handlers.list({ appUid }); // [{ name, hash, updatedAt, subscriptions }] +await puter.events.handlers.remove('indexDocument', { appUid }); +``` + +## Handlers cannot close over anything + +A handler is serialized with `Function.prototype.toString()` and run later, somewhere else. A closed-over variable is not discouraged — it is **unrepresentable**, because nothing around the function survives the trip. + +So every identifier a handler names must be one of: a parameter, something the handler itself declares, a standard global (`fetch`, `JSON`, `Math`, `console`, `URL`, `crypto`, …), or reached through `ctx`. The SDK checks this before the request and rejects with `events_handler_free_variable`, naming the identifier: + +```js +const endpoint = 'https://example.com/ingest'; + +// Rejected: `endpoint` is not a parameter, a local, or a known global. +await puter.events.handlers.publish('ingestUpload', ({ event }) => fetch(endpoint), { appUid }); + +// Accepted: the value travels with the subscription, not with the code. +await puter.events.handlers.publish('ingestUpload', ({ event, ctx }) => fetch(ctx.endpoint), { appUid }); +await puter.events.onPersistent({ subject: 'fs:~/inbox', handlerName: 'ingestUpload', context: { endpoint } }); +``` + +The check is deliberately conservative: anything it cannot resolve is refused with a clear message, rather than accepted and failed on first delivery in production. + +## `publish()` + +```js +puter.events.handlers.publish(name, handler) +puter.events.handlers.publish(name, handler, options) +``` + +- `name` (String) (required): The name subscriptions bind to. Letters, digits and `_ . : -`, starting alphanumeric, up to 128 characters. Unique per app, and stable across source changes. +- `handler` (Function | String | Object) (required): A function (serialized with `toString()`), a source string, or `{ file: '~/AppData/…/handler.js' }`. **A file reference resolves now, not at delivery** — the bytes as they are at this call are what gets deployed, so editing the file afterwards changes nothing until you publish again. +- `options.replace` (Boolean): Take the name whatever is published under it. +- `options.appUid` (String): The app to publish into. Required for an account session. + +Resolves to `{ name, hash, updatedAt, outcome, resumed }`. `outcome` is `'created'`, `'updated'`, or `'unchanged'` when the same source was already published. `resumed` counts subscriptions this publish brought back out of suspension. + +### Two build steps must not silently pick a winner + +The source hash is a change detector and an idempotency key: publishing the **same** source again is a no-op. Publishing **different** source is an update — but only from a caller that knows what it is updating. + +The SDK remembers the hash it last saw published for each name and sends it as the base. A publish whose base has moved under it — a second build step got there first — is refused with `events_handler_conflict`. Pass `replace: true` to say you mean to take the name regardless. + +A client that has never published or listed that name sends no base, so its publish can only create, or be idempotent. + +## `publishAll()` + +```js +puter.events.handlers.publishAll(handlers) +puter.events.handlers.publishAll(handlers, options) +``` + +Publishes a set in one call — what a build step has. `handlers` is an array of `{ name, handler, replace? }`, capped at 50 entries and taken in order. An item the server refuses stops the pass, so a deploy never reports success over a half-published set; items before it are published, and the error names where it stopped. + +Resolves to an array of the same objects `publish()` returns. + +## `list()` + +```js +puter.events.handlers.list() +puter.events.handlers.list(options) +``` + +Resolves to `[{ name, hash, updatedAt, subscriptions }]` for everything the app has published, ordered by name. `subscriptions` counts what is bound to that name, **suspended ones included** — a suspended subscription is still a dependent, and it is the reason removing a name is not just a delete. + +**Source is never returned.** It is the app's own code, read only on the delivery path. + +## `remove()` + +```js +puter.events.handlers.remove(name) +puter.events.handlers.remove(name, options) +``` + +Resolves to `{ name, removed, suspended }`. + +| Situation | What happens | +| --- | --- | +| Nothing is bound to the name | The handler is deleted outright. | +| Subscriptions are bound to it | The handler is deleted **and** every subscription on it is *suspended* with `suspendedReason: 'handler_not_found'` — not deleted. The app's developer is notified. | + +**Publishing the name again resumes them.** That is what makes a bad deploy recoverable: the subscriptions keep their ids, their context and their place, and start delivering again on the next publish. + +Renaming is publish-new plus remove-old, and subscriptions do **not** follow — that is a re-subscribe, deliberately: silently repointing someone's subscription at different code is exactly what consent is protecting against. + +### What a suspension does to the backlog + +A suspended subscription stops being delivered to and stops being metered — so it cannot go on holding a full backlog for free. On suspension its undelivered deliveries are trimmed to **100** and given a deadline: **24 hours** for `handler_not_found` and `failures`, **1 hour** for `no_credit`. Past the deadline they are dropped and one `gap` marker with `reason: 'suspended_backlog_expired'` takes their place, so a resumed subscription learns there were events rather than reading the silence as "nothing changed". A subscription suspended by `permission_revoked` has its backlog **purged at once** and never resumes. + +## Errors + +All four methods reject with `{ message, code }`: + +| `code` | Meaning | +| --- | --- | +| `events_handler_free_variable` | The handler names something it cannot carry. The message names the identifier. | +| `events_handler_invalid` | `handler` is not a function, a source string, or `{ file }`. | +| `events_handler_name_invalid` | The name is empty, too long, or not an addressable identifier. | +| `events_handler_conflict` | Different source is published under this name and the caller did not name it as the base. Pass `replace: true` to take it. | +| `events_handler_app_required` | An account session did not name an app. | +| `events_handler_forbidden` | The caller does not own the app — and an app that is not there answers the same way. | +| `events_handler_too_large` | The serialized handler is over 64 KB. | +| `events_handler_source_invalid` | The handler source is empty. | +| `events_handler_limit` | The app already has the maximum number of published handlers. | +| `too_many_requests` | Over the handler publish/remove budget. | +| `events_disabled` | Events are not enabled on this server. | + +## Examples + +Publish a handler, bind a subscription to it, then take it away + +```html + + + + + + +``` + +Deploy a whole set from a build step + +```html + + + + + + +``` diff --git a/src/docs/src/Events/list.md b/src/docs/src/Events/list.md new file mode 100644 index 000000000..d0278fff8 --- /dev/null +++ b/src/docs/src/Events/list.md @@ -0,0 +1,89 @@ +--- +title: puter.events.list() +description: List the persistent subscriptions this caller holds. +platforms: [websites, apps, nodejs, workers] +--- + +
The Events API is in beta. Event shapes, limits, and behavior may change between releases.
+ +Lists the persistent subscriptions created with [`puter.events.onPersistent()`](/Events/onPersistent/). Session subscriptions made with `onLocal()` are not listed — they live with the connection and are not stored anywhere. + +An app sees only the subscriptions it created. A session acting for the account sees them all, **including ones left behind by an app that is gone** — which is what makes the account the place a stray subscription is revoked from. + +## Syntax +```js +puter.events.list() +puter.events.list(options) +``` + +## Parameters + +#### `options` (Object) (optional) + +- `limit` (Number): Maximum subscriptions per request. Capped at 200; defaults to 50. +- `cursor` (String | null): Continuation token from a previous page. Passing it — `null` included — switches the return value to a single page envelope. +- `includeTotal` (Boolean): Adds `total` to the envelope. Request it on the first page only; it costs more the more subscriptions exist. +- `stream` (Boolean): Returns an async iterator of page envelopes instead of a promise. + +## Return value + +With no pagination params, a `Promise` for an array of every subscription, fetched page by page under the hood. With `cursor` or `includeTotal`, a `Promise` for one page: `{ items, cursor?, total? }` — `cursor` is present only while more pages exist. With `stream: true`, an async iterator of those envelopes. + +**Pages may be short.** Never read `items.length < limit` as the end of the list; iterate until `cursor` is absent. + +Each subscription is the object [`onPersistent()`](/Events/onPersistent/) returns. In particular: + +- `contextKeys` (Array | null) and `contextHash` (String | null) describe the stored `context`. **The values are never returned** — the context is where an API key lives, and a listing is the one surface an app can call repeatedly. The hash changes whenever any value does, which is enough to tell two subscriptions apart or to notice one was re-created. +- `suspendedAt` (Number | null) and `suspendedReason` (String | null) say whether a subscription stopped delivering without being removed, and why: `handler_not_found`, `failures`, `no_credit`, or `permission_revoked`. + +The promise rejects with `{ message, code }` — `too_many_requests` over the listing budget, `events_disabled` where events are off. + +## Examples + +List everything this account is watching + +```html + + + + + + +``` + +Find the ones that stopped, and why + +```html + + + + + + +``` diff --git a/src/docs/src/Events/onPersistent.md b/src/docs/src/Events/onPersistent.md new file mode 100644 index 000000000..75d40376f --- /dev/null +++ b/src/docs/src/Events/onPersistent.md @@ -0,0 +1,156 @@ +--- +title: puter.events.onPersistent() +description: Subscribe to changes with a subscription that keeps running when your app is closed. +platforms: [websites, apps, nodejs, workers] +--- + +
The Events API is in beta. Event shapes, limits, and behavior may change between releases.
+ +Creates a subscription that outlives this connection. It is stored against the account, keeps matching while your app is closed, and runs a handler your app published with [`puter.events.handlers.publish()`](/Events/handlers/). Contrast [`puter.events.onLocal()`](/Events/onLocal/), which lives and dies with the page. + +See [Events](/Events/) for the subject grammar and the event shape. + +## Syntax +```js +puter.events.onPersistent(options) +``` + +## Parameters + +#### `options` (Object) (required) + +- `subject` (String) (required): What to watch — the same grammar `onLocal()` takes, e.g. `fs:~/Documents` or `fs:~/inbox/*.json:add`. +- `delivery` (String): `'broadcast'` (default) delivers to everything listening. `'single'` delivers each event to exactly one consumer, which must acknowledge it, and requires `handlerName`. +- `targets` (Array): Transports deliveries may take — any of `'socket'`, `'worker'`, `'push'`. Defaults to `['socket', 'worker']` for a subscription an app made, `['socket']` for one an account session made naming no app. A `single` subscription may not target `'push'`; a subscription with no app may not target `'worker'` — there is exactly one events worker per app, and no app means no worker to invoke. +- `handlerName` (String): The published handler this subscription binds to. Required for `single`. +- `handler` (Function | String | Object): The handler source this subscription was written against. Sent as a **hash**, never as source: the subscription binds only if that hash matches what is published under `handlerName`, which is why `handlerName` is required alongside it. Accepts a function, a source string, or `{ file: '~/AppData/…/handler.js' }`. +- `context` (Object): Values the handler needs, delivered to it as a frozen `ctx`. **Capped at 4 KB serialized** — see below. +- `expiresAt` (Number | String): When the subscription ends by itself — unix seconds or an ISO-8601 string, and it has to be in the future. + +## `context` is evaluated once, and capped at 4 KB + +A handler is deployed, not called: it is serialized and run later, somewhere else, so it cannot close over anything. `context` is how values reach it — and it is evaluated **at this call**, serialized, and never re-evaluated. `ctx.endpoint` is whatever `process.env.INGEST_URL` was when you subscribed, forever, until you subscribe again. + +```js +await puter.events.onPersistent({ + subject: 'fs:~/inbox', + handlerName: 'ingestUpload', + context: { endpoint: process.env.INGEST_URL, apiKey: process.env.INGEST_KEY }, +}); +``` + +**The cap is a hard 4 KB.** These are database rows read on every delivery, and `context` is the one field you control the size of; over the cap the call fails with `events_context_too_large`, client-side, before the request. Context is stored in plaintext and is read only on the delivery path — [`puter.events.list()`](/Events/list/) returns its **key names and a content hash**, never its values. If you need to hand a handler more than 4 KB, put it in a file and pass the path in `context`; a wider column is not the upgrade path. + +## Return value + +A `Promise` that resolves to the subscription: + +- `subId` (String): Its id, and what [`puter.events.unsubscribe()`](/Events/unsubscribe/) names. Stable for the life of the subscription. +- `subject`, `anchor`, `match`, `op`: as `onLocal()` returns them. +- `delivery` (String), `targets` (Array), `handlerName` (String | null). +- `appUid` (String | null): The app that created it, or `null` for one an account session made. +- `contextKeys` (Array | null), `contextHash` (String | null): the shape of the stored context, never its values. +- `createdAt`, `expiresAt` (Number | null): unix seconds. +- `suspendedAt` (Number | null), `suspendedReason` (String | null): why it stopped delivering without being removed — see [`puter.events.handlers.remove()`](/Events/handlers/). + +The promise rejects with `{ message, code }`: + +| `code` | Meaning | +| --- | --- | +| `invalid_subject` | The subject is not a non-empty string, or the server could not parse it. | +| `events_handler_name_required` | An inline `handler` was given with no `handlerName` to publish it under. | +| `events_handler_free_variable` | The handler names something it cannot carry — a closed-over variable. The message names the identifier. | +| `events_handler_invalid` | `handler` is not a function, a source string, or `{ file }`. | +| `events_handler_hash_unavailable` | This environment provides no `crypto.subtle`, so an inline handler cannot be hashed. Publish it first and pass `handlerName` alone. | +| `events_handler_not_found` | No handler is published under `handlerName`. The subscription is **not** created. | +| `events_handler_hash_mismatch` | The published handler is not the source this subscription was written against. | +| `events_handler_required` | `delivery: 'single'` without a `handlerName`. | +| `events_context_too_large` | The serialized `context` is over 4 KB. | +| `events_context_invalid` | `context` is not JSON-serializable. | +| `invalid_targets` | A target outside `socket`/`worker`/`push`, `push` on a `single` subscription, or `worker` on a subscription with no app. | +| `invalid_expires_at` | `expiresAt` is not a future time. | +| `subject_does_not_exist` | The subject is not there, or this account cannot read it. | +| `events_subscription_limit` | This account already holds the maximum number of persistent subscriptions. | +| `too_many_requests` | Over the subscribe/unsubscribe call budget. | +| `events_disabled` | Events are not enabled on this server. | + +## Examples + +Watch a folder with a handler that keeps running + +```html + + + + + + +``` + +Bind to the exact source you wrote against + +```html + + + + + + +``` diff --git a/src/docs/src/Events/unsubscribe.md b/src/docs/src/Events/unsubscribe.md new file mode 100644 index 000000000..4a6bfdedf --- /dev/null +++ b/src/docs/src/Events/unsubscribe.md @@ -0,0 +1,66 @@ +--- +title: puter.events.unsubscribe() +description: End a persistent subscription. +platforms: [websites, apps, nodejs, workers] +--- + +
The Events API is in beta. Event shapes, limits, and behavior may change between releases.
+ +Ends a subscription created with [`puter.events.onPersistent()`](/Events/onPersistent/). It stops matching immediately and everything it was still owed goes with it — a backlog held for a subscription nobody can consume is memory, and the paths it names are ones its holder just stopped asking about. + +For a session subscription made with [`puter.events.onLocal()`](/Events/onLocal/), use [`subscription.off()`](/Events/off/) instead. + +## Syntax +```js +puter.events.unsubscribe(subId) +``` + +## Parameters + +#### `subId` (String) (required) +The `subId` of the subscription to end, as `onPersistent()` returned it or as [`puter.events.list()`](/Events/list/) reports it. + +## Return value + +A `Promise` that resolves when the subscription is gone. + +An id this caller does not hold — one already ended, or one another app created — **reads as absent** rather than refused, so the call cannot be used to find out which subscriptions exist. It rejects with `{ message, code }`: + +| `code` | Meaning | +| --- | --- | +| `subscription_does_not_exist` | No such subscription, or not this caller's. | +| `too_many_requests` | Over the subscribe/unsubscribe call budget. | +| `events_disabled` | Events are not enabled on this server. | + +An app may only end the subscriptions it created. A session acting for the account may end any of them, including ones left behind by an app that is gone. + +## Examples + +Create a persistent subscription, then end it + +```html + + + + + + +``` diff --git a/src/docs/src/rate-limits-and-quotas.md b/src/docs/src/rate-limits-and-quotas.md index 2d56dd9ef..08d1fcfb9 100644 --- a/src/docs/src/rate-limits-and-quotas.md +++ b/src/docs/src/rate-limits-and-quotas.md @@ -170,15 +170,34 @@ One write can reach many subscriptions, so events are bounded on both halves: ho | Deliveries per minute, per subscription | 600 | | Acknowledgements per minute | 600 | | Undelivered deliveries per subscription | 10,000 | +| Undelivered deliveries per *suspended* subscription | 100 | | Suspended subscriptions kept for | 30 days | +| Published handlers per app | 100 | +| Handler source size | 64 KB | +| Handlers per `publishAll` call | 50 | +| Handler publish / remove calls per minute | 60 | +| Handler listings per minute | 120 | Subscriptions come in two kinds. A **session** subscription lives with the connection that made it: it is dropped when the connection closes, and a reconnecting client subscribes again. A **durable** subscription outlives every connection — it is created over the API, listed and revoked from the account, and keeps delivering until you remove it or it expires. The 51st subscription on one connection, and the 501st durable subscription on one account, both fail with `events_subscription_limit`. Over the call budget, `subscribe` and `unsubscribe` fail with `too_many_requests`. Subscribing to something you cannot read fails with `subject_does_not_exist` — the same answer as subscribing to something that is not there, so the call cannot be used to find out which. -A durable subscription may carry a `context`: JSON that is stored with it and handed to its handler on every delivery, capped at **4 KB** and rejected over that with `events_context_too_large`. Listings never return it. An app sees and revokes only the subscriptions it created; a session acting for the account sees them all, including ones left behind by an app that has since been removed. +A durable subscription may carry a `context`: JSON that is stored with it and handed to its handler on every delivery, capped at a hard **4 KB** and rejected over that with `events_context_too_large` — client-side, before the request. It is stored in plaintext and read only on the delivery path; listings return its **key names and a content hash**, never its values. For anything larger, store it in a file and put the path in `context`. An app sees and revokes only the subscriptions it created; a session acting for the account sees them all, including ones left behind by an app that has since been removed. -**A subscription can end without you unsubscribing.** Access is re-checked against the stored permission on every delivery, so a share that is taken back stops delivering immediately; the subscription is then *suspended*, with `suspendedAt` and `suspendedReason: 'permission_revoked'` in `list` and a notification to whoever holds it. The same happens to every subscription an app holds for you when you withdraw that app's access. Re-granting does not bring a suspended subscription back — subscribe again, which is how consent to watch is re-established — and a suspended row is deleted **30 days** after it stops. Deleting the node a subscription is anchored on ends it too, unless the subject named a path or a pattern, in which case it follows that path up to the nearest folder that still exists and keeps watching, so recreating the path resumes delivery. +A durable subscription runs a **handler** its app published by name. An app may publish **100** of them, each up to **64 KB** of source, and a name is unique inside one app. Publishing is a developer operation: the account has to own the app. Publishing the same source again is a no-op; publishing different source under a name whose current source the caller did not name as its base is refused with `events_handler_conflict`, so two racing build steps never silently pick a winner — `replace: true` is how a caller says it means to take the name. Handler source is never returned by any listing. + +**A subscription can end or stop without you unsubscribing.** Access is re-checked against the stored permission on every delivery, so a share that is taken back stops delivering immediately; the subscription is then *suspended*, with `suspendedAt` and `suspendedReason` in `list`. There are four reasons: + +| `suspendedReason` | Cause | Resumes when | +| --- | --- | --- | +| `handler_not_found` | The handler it is bound to was removed | The name is published again | +| `failures` | Its handler failed or timed out repeatedly | The subscription is republished against a working handler | +| `no_credit` | Its holder ran out of credit | The balance is restored | +| `permission_revoked` | The grant it was made under was withdrawn | **Never** — subscribe again | + +A suspended subscription stops delivering and stops being metered, so it cannot go on holding a full backlog for free: what it is owed is trimmed to **100** deliveries and given a deadline — **24 hours** for `handler_not_found` and `failures`, **1 hour** for `no_credit` — after which they are dropped and one `gap` marker with `reason: 'suspended_backlog_expired'` takes their place. A subscription suspended by `permission_revoked` has its backlog **purged immediately**: it names paths its holder has just lost the right to see, and holding them for a resume that by design never comes would turn a revocation into a delayed disclosure. A suspended row itself is deleted **30 days** after it stops. + +Deleting the node a subscription is anchored on ends it too, unless the subject named a path or a pattern, in which case it follows that path up to the nearest folder that still exists and keeps watching, so recreating the path resumes delivery. Match patterns are compiled once when you subscribe and are capped at **256 characters** and **16 segments**; anything larger is rejected with `invalid_subject_pattern`. `**` crosses directories and costs no more than `*`. diff --git a/src/docs/src/sidebar.js b/src/docs/src/sidebar.js index 11ad84870..11e4ebc39 100755 --- a/src/docs/src/sidebar.js +++ b/src/docs/src/sidebar.js @@ -422,6 +422,38 @@ let sidebar = [ source: '/Events/off.md', path: '/Events/off', }, + { + title: 'onPersistent()', + page_title: 'puter.events.onPersistent()', + title_tag: 'puter.events.onPersistent()', + icon: '/assets/img/function.svg', + source: '/Events/onPersistent.md', + path: '/Events/onPersistent', + }, + { + title: 'list()', + page_title: 'puter.events.list()', + title_tag: 'puter.events.list()', + icon: '/assets/img/function.svg', + source: '/Events/list.md', + path: '/Events/list', + }, + { + title: 'unsubscribe()', + page_title: 'puter.events.unsubscribe()', + title_tag: 'puter.events.unsubscribe()', + icon: '/assets/img/function.svg', + source: '/Events/unsubscribe.md', + path: '/Events/unsubscribe', + }, + { + title: 'handlers', + page_title: 'puter.events.handlers', + title_tag: 'puter.events.handlers', + icon: '/assets/img/function.svg', + source: '/Events/handlers.md', + path: '/Events/handlers', + }, ], }, { diff --git a/src/puter-js/index.d.ts b/src/puter-js/index.d.ts index e54454d79..188e37198 100644 --- a/src/puter-js/index.d.ts +++ b/src/puter-js/index.d.ts @@ -98,11 +98,18 @@ export type { EventDelivery, EventGapMarker, EventHandler, + HandlerOptions, + HandlerPublication, + HandlerSummary, OnLocalOptions, + OnPersistentOptions, + PersistentSubscription, + PublishedHandler, PuterEvent, PuterKvEvent, } from './types/modules/events/types.js'; export type { EventSubscription } from './types/modules/events/lib/subscription.js'; +export type { EventHandlers } from './types/modules/events/lib/handlers.js'; // -- puter.fs -- export type { diff --git a/src/puter-js/src/modules/events/index.js b/src/puter-js/src/modules/events/index.js index 3fe87f48f..750254512 100644 --- a/src/puter-js/src/modules/events/index.js +++ b/src/puter-js/src/modules/events/index.js @@ -1,6 +1,10 @@ import { PuterModule } from '../../lib/PuterModule.js'; import { EventChannel } from './lib/channel.js'; +import { EventHandlers } from './lib/handlers.js'; +import { list } from './list.js'; import { onLocal } from './onLocal.js'; +import { onPersistent } from './onPersistent.js'; +import { unsubscribe } from './unsubscribe.js'; /** @typedef {import('../../index.js').Puter} Puter */ @@ -9,15 +13,23 @@ import { onLocal } from './onLocal.js'; * path that does not exist yet — and a handler runs whenever something under * it changes. * + * Two kinds of subscription: `onLocal()` lives with this connection and is + * gone when the page is, while `onPersistent()` is stored against the account + * and keeps matching with nothing open. A persistent subscription runs a + * handler the app published through `puter.events.handlers`. + * * Method implementations live in the sibling files as `this`-context functions * whose JSDoc is the source of truth for the public signatures — `types/` is * generated from it, never edited by hand. */ export class EventsModule extends PuterModule { - // The field holds the unbound function so it keeps its full type (`bind` - // erases overloads); the constructor rebinds it so destructured calls - // (`const { onLocal } = puter.events`) work. + // The fields hold the unbound functions so they keep their full types + // (`bind` erases overloads); the constructor rebinds them so destructured + // calls (`const { onLocal } = puter.events`) work. onLocal = onLocal; + onPersistent = onPersistent; + unsubscribe = unsubscribe; + list = list; /** @param {Puter} puter */ constructor (puter) { @@ -29,10 +41,15 @@ export class EventsModule extends PuterModule { */ this.channel = new EventChannel(this); + /** The named functions this app has deployed. */ + this.handlers = new EventHandlers(this); + const methods = /** @type {Record unknown>} */ ( /** @type {unknown} */ (this) ); - methods.onLocal = methods.onLocal.bind(this); + for ( const name of ['onLocal', 'onPersistent', 'unsubscribe', 'list'] ) { + methods[name] = methods[name].bind(this); + } // The socket carries its token from the moment it connects, so a new // token means a new connection — and the subscriptions on the old one diff --git a/src/puter-js/src/modules/events/lib/api.js b/src/puter-js/src/modules/events/lib/api.js new file mode 100644 index 000000000..ead31617c --- /dev/null +++ b/src/puter-js/src/modules/events/lib/api.js @@ -0,0 +1,64 @@ +import { fetchUrl } from '../../../lib/networkUtils.js'; +import { PuterJSError } from '../../../lib/PuterJSError.js'; + +/** + * The HTTP half of `puter.events`. The socket verbs carry session + * subscriptions; everything that outlives a connection — durable + * subscriptions and the handlers they bind — is a route. + * + * The server's `{ message, code }` is passed through untouched: its codes are + * the API surface callers branch on, and re-wrapping them here would make the + * SDK a second place they are defined. + */ + +/** The failure shape for a response that carried no usable body. */ +const requestFailed = (status) => + new PuterJSError( + `The events request failed (HTTP ${status})`, + 'events_failed', + ); + +/** + * @param {import('../../../index.js').Puter} puter + * @param {string} route + * @param {Record} [body] Present makes it a POST. + * @param {Record} [query] + * @returns {Promise>} + */ +export async function request (puter, route, body, query) { + const search = new URLSearchParams(); + for ( const [key, value] of Object.entries(query ?? {}) ) { + if ( value === undefined || value === null ) continue; + search.set(key, String(value)); + } + const suffix = search.toString(); + + const response = await fetchUrl( + `${puter.APIOrigin}${route}${suffix ? `?${suffix}` : ''}`, + { + method: body ? 'POST' : 'GET', + includePuterAuth: true, + headers: { 'Content-Type': 'application/json' }, + ...(body ? { body: JSON.stringify(body) } : {}), + }, + ); + + const isJson = response.headers.get('content-type')?.includes('application/json'); + const parsed = isJson ? await response.json() : null; + + if ( response.status !== 200 ) { + if ( ! parsed ) throw requestFailed(response.status); + const { message, error, code, ...rest } = parsed; + throw new PuterJSError( + typeof message === 'string' + ? message + : typeof error === 'string' + ? error + : `The events request failed (HTTP ${response.status})`, + typeof code === 'string' ? code : 'events_failed', + rest, + ); + } + + return parsed ?? {}; +} diff --git a/src/puter-js/src/modules/events/lib/channel.js b/src/puter-js/src/modules/events/lib/channel.js index 676ec3836..a2f155d25 100644 --- a/src/puter-js/src/modules/events/lib/channel.js +++ b/src/puter-js/src/modules/events/lib/channel.js @@ -328,7 +328,10 @@ export class EventChannel { // An event for something this client has already unsubscribed from: // in flight when `off()` was called, and no longer anybody's. if ( ! sub || ! envelope.event ) return; - sub.deliver(/** @type {PuterEvent | PuterKvEvent | EventGapMarker} */ (envelope.event)); + sub.deliver( + /** @type {PuterEvent | PuterKvEvent | EventGapMarker} */ (envelope.event), + /** @type {Record | undefined} */ (envelope.ctx), + ); } /** diff --git a/src/puter-js/src/modules/events/lib/freeVariables.js b/src/puter-js/src/modules/events/lib/freeVariables.js new file mode 100644 index 000000000..ae47d3c2d --- /dev/null +++ b/src/puter-js/src/modules/events/lib/freeVariables.js @@ -0,0 +1,311 @@ +import { PuterJSError } from '../../../lib/PuterJSError.js'; +import { tokenize } from './tokenize.js'; + +/** + * The free-variable scan a handler is held to at subscribe time. + * + * A handler is deployed, not called: it is serialized with + * `Function.prototype.toString()` and run later, somewhere else, with nothing + * around it. A closed-over variable is therefore not discouraged, it is + * unrepresentable — so anything the source names that it does not also bind has + * to come from `ctx`, from a parameter, or from the runtime. Catching that here + * turns a rule that would otherwise fail on the first delivery, in production, + * into a rejected `subscribe`. + * + * The scan collects every name the source *binds* anywhere — parameters, + * destructured names, `var`/`let`/`const`/`function`/`class`/`catch` — and then + * requires every identifier *reference* to be one of those or a known global. + * + * Known limitation: bindings are collected flat rather than per scope, so a + * name bound in one block counts as bound in the whole handler. That direction + * is deliberate — it can miss a shadowing case, and it never rejects code that + * would have worked. + */ + +/** + * Reserved words and the contextual keywords that read as identifiers. Skipped + * rather than resolved: none of them is a variable reference, and treating + * `async` or `get` as one would reject perfectly ordinary handlers. + */ +const KEYWORDS = new Set([ + 'await', 'break', 'case', 'catch', 'class', 'const', 'continue', 'debugger', + 'default', 'delete', 'do', 'else', 'enum', 'export', 'extends', 'false', + 'finally', 'for', 'function', 'if', 'import', 'in', 'instanceof', 'let', + 'new', 'null', 'return', 'super', 'switch', 'this', 'throw', 'true', 'try', + 'typeof', 'var', 'void', 'while', 'with', 'yield', + 'async', 'as', 'from', 'get', 'set', 'of', 'static', 'accessor', +]); + +/** Declaration keywords whose head is a binding pattern. */ +const DECLARATORS = new Set(['var', 'let', 'const']); + +/** + * Names the runtime provides. Curated rather than derived from `globalThis`: + * the handler runs in a worker isolate, not in the environment doing the scan, + * so what is present here says nothing about what is present there. + */ +export const HANDLER_GLOBALS = new Set([ + // Language + 'globalThis', 'undefined', 'NaN', 'Infinity', 'arguments', + 'Object', 'Array', 'String', 'Number', 'Boolean', 'Symbol', 'BigInt', + 'Math', 'JSON', 'Date', 'RegExp', 'Function', 'Promise', 'Proxy', 'Reflect', + 'Map', 'Set', 'WeakMap', 'WeakSet', 'WeakRef', 'FinalizationRegistry', + 'Error', 'TypeError', 'RangeError', 'SyntaxError', 'ReferenceError', + 'EvalError', 'URIError', 'AggregateError', 'Intl', + 'ArrayBuffer', 'SharedArrayBuffer', 'DataView', + 'Int8Array', 'Uint8Array', 'Uint8ClampedArray', 'Int16Array', 'Uint16Array', + 'Int32Array', 'Uint32Array', 'Float32Array', 'Float64Array', + 'BigInt64Array', 'BigUint64Array', + 'parseInt', 'parseFloat', 'isNaN', 'isFinite', + 'encodeURI', 'encodeURIComponent', 'decodeURI', 'decodeURIComponent', + 'structuredClone', 'queueMicrotask', 'atob', 'btoa', + // Runtime + 'console', 'fetch', 'Request', 'Response', 'Headers', 'FormData', 'Blob', + 'File', 'URL', 'URLSearchParams', 'AbortController', 'AbortSignal', + 'TextEncoder', 'TextDecoder', 'ReadableStream', 'WritableStream', + 'TransformStream', 'CompressionStream', 'DecompressionStream', + 'crypto', 'Crypto', 'SubtleCrypto', 'performance', 'WebSocket', + 'Event', 'EventTarget', 'CustomEvent', 'MessageChannel', 'MessagePort', + 'setTimeout', 'clearTimeout', 'setInterval', 'clearInterval', + // The SDK the worker runs inside. + 'puter', +]); + +/** Raised for the identifier that could not be resolved, naming it. */ +const freeVariable = (name) => + new PuterJSError( + `Handler refers to \`${name}\`, which is not a parameter, a local, or a known global. ` + + 'A handler is serialized and run elsewhere, so it cannot close over anything — ' + + 'pass the value in `context` and read it from `ctx`.', + 'events_handler_free_variable', + ); + +const isName = (token) => token?.type === 'name'; +const isPunct = (token, value) => token?.type === 'punct' && token.value === value; + +const OPENERS = { '(': ')', '[': ']', '{': '}' }; +const CLOSERS = new Set([')', ']', '}']); + +/** Index of the token closing the group that opens at `start`, or -1. */ +const matchGroup = (tokens, start) => { + const stack = []; + for ( let i = start; i < tokens.length; i++ ) { + const token = tokens[i]; + if ( token.type !== 'punct' ) continue; + if ( OPENERS[token.value] ) { stack.push(OPENERS[token.value]); continue; } + if ( ! CLOSERS.has(token.value) ) continue; + if ( stack.pop() !== token.value ) return -1; + if ( stack.length === 0 ) return i; + } + return -1; +}; + +/** + * Collect the names a binding pattern introduces, between `start` and `end`. + * Everything after an `=` is an initializer — a reference, not a binding — so + * it is skipped until the comma that ends that binder. + */ +const collectPattern = (tokens, start, end, into) => { + let depth = 0; + let inInitializer = false; + for ( let i = start; i < end; i++ ) { + const token = tokens[i]; + if ( token.type === 'punct' ) { + if ( OPENERS[token.value] ) depth++; + else if ( CLOSERS.has(token.value) ) depth--; + else if ( token.value === '=' ) inInitializer = true; + else if ( token.value === ',' && depth <= 0 ) inInitializer = false; + continue; + } + if ( inInitializer || ! isName(token) || KEYWORDS.has(token.value) ) continue; + // `.b` in a pattern is a member target, which binds nothing new. + if ( isPunct(tokens[i - 1], '.') ) continue; + into.add(token.value); + } +}; + +/** Names a `var`/`let`/`const` head introduces, and where the head ends. */ +const collectDeclaration = (tokens, start, into) => { + let depth = 0; + let i = start; + let inInitializer = false; + for ( ; i < tokens.length; i++ ) { + const token = tokens[i]; + if ( token.type === 'punct' ) { + if ( OPENERS[token.value] ) { depth++; continue; } + if ( CLOSERS.has(token.value) ) { + if ( depth === 0 ) return i; + depth--; + continue; + } + if ( depth > 0 ) { + if ( token.value === '=' ) inInitializer = true; + else if ( token.value === ',' ) inInitializer = false; + continue; + } + if ( token.value === ';' ) return i; + if ( token.value === '=' ) inInitializer = true; + else if ( token.value === ',' ) inInitializer = false; + continue; + } + if ( isName(token) && depth === 0 && (token.value === 'of' || token.value === 'in') ) + return i; + if ( inInitializer || ! isName(token) || KEYWORDS.has(token.value) ) continue; + if ( isPunct(tokens[i - 1], '.') ) continue; + into.add(token.value); + } + return i; +}; + +/** + * Every name the source binds, wherever it binds it. Over-approximate on + * purpose: the alternative is a scope tree, and the cost of getting one wrong + * is rejecting a handler that works. + * + * @param {import('./tokenize.js').Token[]} tokens + * @returns {Set} + */ +export const collectBindings = (tokens) => { + /** @type {Set} */ + const bound = new Set(); + + for ( let i = 0; i < tokens.length; i++ ) { + const token = tokens[i]; + + if ( isPunct(token, '=>') ) { + const before = tokens[i - 1]; + if ( isPunct(before, ')') ) { + // Walk back to the `(` this `)` closes. + let depth = 0; + for ( let j = i - 1; j >= 0; j-- ) { + const back = tokens[j]; + if ( back.type !== 'punct' ) continue; + if ( CLOSERS.has(back.value) ) depth++; + else if ( OPENERS[back.value] ) { + depth--; + if ( depth === 0 ) { + collectPattern(tokens, j + 1, i - 1, bound); + break; + } + } + } + } else if ( isName(before) && ! KEYWORDS.has(before.value) ) { + bound.add(before.value); + } + continue; + } + + if ( ! isName(token) ) continue; + + if ( DECLARATORS.has(token.value) ) { + i = collectDeclaration(tokens, i + 1, bound) - 1; + continue; + } + + if ( token.value === 'function' || token.value === 'class' ) { + const next = tokens[i + 1]; + // `function *gen()` and `function ()` both leave the name absent. + const nameAt = isPunct(next, '*') ? i + 2 : i + 1; + if ( isName(tokens[nameAt]) && ! KEYWORDS.has(tokens[nameAt].value) ) + bound.add(tokens[nameAt].value); + continue; + } + + if ( token.value === 'catch' && isPunct(tokens[i + 1], '(') ) { + const close = matchGroup(tokens, i + 1); + if ( close !== -1 ) collectPattern(tokens, i + 2, close, bound); + continue; + } + + // `name(...) {` is a function or method definition — every construct + // that reads the same way (`if`, `for`, `while`, `switch`, `catch`) is + // a keyword and never reaches here. Its parameters are bindings, and so + // is the name itself. + if ( ! KEYWORDS.has(token.value) && isPunct(tokens[i + 1], '(') ) { + const close = matchGroup(tokens, i + 1); + if ( close !== -1 && isPunct(tokens[close + 1], '{') ) { + bound.add(token.value); + collectPattern(tokens, i + 2, close, bound); + } + continue; + } + } + + // An anonymous `function (a, b) {`, whose parameters the pass above only + // reaches when the function is named. + for ( let i = 0; i < tokens.length; i++ ) { + if ( ! isName(tokens[i]) || tokens[i].value !== 'function' ) continue; + let open = i + 1; + while ( open < tokens.length && ! isPunct(tokens[open], '(') ) { + if ( isPunct(tokens[open], '{') ) break; + open++; + } + if ( ! isPunct(tokens[open], '(') ) continue; + const close = matchGroup(tokens, open); + if ( close !== -1 ) collectPattern(tokens, open + 1, close, bound); + } + + return bound; +}; + +/** + * Identifiers the source *reads*, in order and without duplicates. Property + * names, keys and labels are not reads: `a.b` reaches `b` through `a`, and only + * `a` has to resolve to anything. + * + * @param {import('./tokenize.js').Token[]} tokens + * @returns {string[]} + */ +export const collectReferences = (tokens) => { + const seen = new Set(); + /** @type {string[]} */ + const names = []; + + for ( let i = 0; i < tokens.length; i++ ) { + const token = tokens[i]; + if ( ! isName(token) || KEYWORDS.has(token.value) ) continue; + + const before = tokens[i - 1]; + const after = tokens[i + 1]; + + // `a.b`, `a?.b`, `#private`, and the target of `break`/`continue`. + if ( isPunct(before, '.') || isPunct(before, '?.') || isPunct(before, '#') ) + continue; + if ( isName(before) && (before.value === 'break' || before.value === 'continue') ) + continue; + // A property key or a label. Also swallows the middle of a ternary, + // which is a name this scan then does not check — the safe direction. + if ( isPunct(after, ':') ) continue; + // A method or function definition, whose name is not a read. + if ( isPunct(after, '(') ) { + const close = matchGroup(tokens, i + 1); + if ( close !== -1 && isPunct(tokens[close + 1], '{') ) continue; + } + + if ( seen.has(token.value) ) continue; + seen.add(token.value); + names.push(token.value); + } + + return names; +}; + +/** + * Throws for the first identifier a handler names and cannot reach. Returns the + * bound names, which is only useful to a test. + * + * @param {string} source Serialized handler source. + * @returns {{ bound: Set, references: string[] }} + */ +export const scanHandlerSource = (source) => { + const tokens = tokenize(source); + const bound = collectBindings(tokens); + const references = collectReferences(tokens); + + for ( const name of references ) { + if ( bound.has(name) || HANDLER_GLOBALS.has(name) ) continue; + throw freeVariable(name); + } + + return { bound, references }; +}; diff --git a/src/puter-js/src/modules/events/lib/freeVariables.test.js b/src/puter-js/src/modules/events/lib/freeVariables.test.js new file mode 100644 index 000000000..cc47c19af --- /dev/null +++ b/src/puter-js/src/modules/events/lib/freeVariables.test.js @@ -0,0 +1,181 @@ +import { describe, expect, it } from 'vitest'; +import { scanHandlerSource } from './freeVariables.js'; +import { tokenize } from './tokenize.js'; + +const scan = (source) => scanHandlerSource(source); + +const rejects = (source) => { + try { + scan(source); + } catch (error) { + return error; + } + throw new Error(`expected a rejection for: ${source}`); +}; + +/** Handlers a developer would plausibly write, none of which close over anything. */ +const ACCEPTED = [ + [ + 'the design`s example handler', + `async ({ event, ctx, user, fetch, ack }) => { + await fetch(ctx.endpoint, { + method: 'POST', + body: JSON.stringify({ path: event.path, key: ctx.apiKey }), + }); + await ack(); + }`, + ], + ['a bare arrow with one parameter', 'delivery => console.log(delivery.event.op)'], + ['a named function declaration', 'function onWrite ({ event }) { console.log(event.uid); }'], + ['an anonymous function expression', 'function ({ event, ctx }) { return ctx.prefix + event.path; }'], + ['locals declared with const and let', '({ event }) => { const p = event.path; let n = p.length; return n; }'], + ['a destructured local with a default', '({ ctx }) => { const { retries = 3, url } = ctx; return url.repeat(retries); }'], + ['an array destructuring local', '({ event }) => { const [head, ...rest] = event.path.split("/"); return rest.concat(head); }'], + ['a for-of loop variable', '({ ctx }) => { for (const item of ctx.items) console.log(item); }'], + ['a classic for loop', '({ ctx }) => { for (let i = 0; i < ctx.n; i++) console.log(i); }'], + ['a catch parameter', '({ ctx }) => { try { JSON.parse(ctx.body); } catch (err) { console.warn(err); } }'], + ['a nested function and its parameters', '({ event }) => { const f = (a, b) => a + b; return f(1, event.seq); }'], + ['a class declaration with methods', '({ ctx }) => { class Sink { constructor (url) { this.url = url; } send (body) { return fetch(this.url, { body }); } } return new Sink(ctx.url); }'], + ['object property keys that share a name with nothing', '({ event }) => ({ endpoint: event.path, retries: 2 })'], + ['a template literal reading only ctx', '({ ctx, event }) => `${ctx.base}/${event.uid}`'], + ['a regex literal that looks like division', '({ event }) => /\\/tmp\\/[a-z]+/.test(event.path)'], + ['a comment naming something undeclared', '({ event }) => { /* endpoint is gone now */ return event.uid; }'], + ['a string naming something undeclared', '({ event }) => event.path + "endpoint"'], + ['runtime globals', '({ event }) => { console.log(Date.now(), Math.max(1, event.seq), JSON.stringify(event), new URL("https://x.example")); }'], + ['the SDK global a worker runs inside', '({ user }) => user.puter.fs.read("/x").then(r => puter.print(r))'], + ['optional chaining and computed member access', '({ event, ctx }) => event?.meta?.[ctx.key]'], + ['a shorthand method on an object literal', '({ event }) => ({ run (x) { return x + event.seq; } })'], + ['an async generator with a yield', 'async function* ({ ctx }) { yield ctx.first; }'], + ['a label and a break to it', '({ ctx }) => { outer: for (const a of ctx.rows) { break outer; } }'], + ['a label and a continue to it', '({ ctx, event }) => { loop: while (ctx.n-- > 0) { if (ctx.skip) continue loop; event.push(ctx.n); } }'], + ['a getter on a class', '({ ctx }) => { class C { get url () { return ctx.url; } } return new C(); }'], + [ + 'the design doc`s own example, verbatim', + `async ({ event, ctx, user, fetch, ack }) => { + const meta = await user.fs.stat(event.path); + if (meta.size < ctx.minSize) return ack(); + await fetch(ctx.endpoint, { + method: 'POST', + body: JSON.stringify({ uid: event.uid, size: meta.size }), + }); + await ack(); + }`, + ], + ['object shorthand naming a declared local', '({ event }) => { const endpoint = event.path; return { endpoint }; }'], + ['rest in a destructured object parameter', 'async ({ event, ...rest }) => { return rest.foo + event.seq; }'], + ['typeof on a bound parameter', '({ event }) => typeof event === "object"'], + // Regex directly after a block-closing `}`, with no `return`/other + // regex-triggering keyword in between — the tokenizer has to decide this + // is a regex from the `}` alone, not from what came before it. + [ + 'a regex literal right after a closed block, not division', + '({ ctx }) => { if (ctx.on) { console.log(ctx.on); } /ab+c/.test(ctx.body); }', + ], +]; + +/** Handlers that close over something the serialized source cannot carry. */ +const REJECTED = [ + ['a closure over an outer const', '({ event }) => fetch(endpoint, { body: event.path })', 'endpoint'], + ['a closure used as a bare value', '({ event }) => event.path + suffix', 'suffix'], + ['a closure inside a template hole', '({ event }) => `${base}/${event.uid}`', 'base'], + ['a closure inside a nested function', '({ event }) => { const f = () => apiKey; return f(); }', 'apiKey'], + ['a closure used as a call target', '({ event }) => publish(event)', 'publish'], + ['a closure in a default parameter value', '({ event }, retries = maxRetries) => retries + event.seq', 'maxRetries'], + ['a closure in a destructuring default', '({ event, timeout = defaultTimeout }) => timeout + event.seq', 'defaultTimeout'], + ['a closure in a for-of subject', '() => { for (const row of rows) console.log(row); }', 'rows'], + ['a closure used with new', '({ ctx }) => new Sink(ctx.url)', 'Sink'], + ['a closure in a declaration initializer', '({ event }) => { const target = destination; return target + event.uid; }', 'destination'], + ['a closure in an object value position', '({ event }) => ({ endpoint: outerEndpoint, path: event.path })', 'outerEndpoint'], + ['a closure in a computed key', '({ event }) => ({ [outerKey]: event.uid })', 'outerKey'], + // Shorthand `{ endpoint }` is sugar for `{ endpoint: endpoint }` — a + // *reference*, not a key — and has to be told apart from `{ endpoint: x }` + // above, where `endpoint` is a label nothing needs to resolve. + ['a closure read through object shorthand', '({ event }) => ({ endpoint, path: event.path })', 'endpoint'], + ['typeof on an undeclared name', '({ event }) => typeof missingGlobal === "undefined" ? event.seq : 0', 'missingGlobal'], +]; + +describe('handlers a scan accepts', () => { + it.each(ACCEPTED)('accepts %s', (_label, source) => { + expect(() => scan(source)).not.toThrow(); + }); +}); + +describe('handlers a scan rejects', () => { + it.each(REJECTED)('rejects %s', (_label, source, identifier) => { + const error = rejects(source); + expect(error.code).toBe('events_handler_free_variable'); + expect(error.message).toContain(`\`${identifier}\``); + }); + + it('names the identifier so the developer knows what to move into context', () => { + const error = rejects('({ event }) => fetch(ingestUrl, { body: event.path })'); + expect(error.message).toContain('`ingestUrl`'); + expect(error.message).toContain('ctx'); + }); +}); + +describe('what the tokenizer hides from the scan', () => { + it('drops strings, comments and regex bodies', () => { + const values = tokenize( + '({ a }) => { /* comment */ const s = "text"; return /pattern/.test(s) && a; }', + ).map(token => token.value); + + expect(values).not.toContain('comment'); + expect(values).not.toContain('text'); + expect(values).not.toContain('pattern'); + expect(values).toContain('a'); + }); + + it('keeps the code inside a template hole', () => { + const values = tokenize('`prefix ${value} suffix`').map(token => token.value); + expect(values).toContain('value'); + expect(values).not.toContain('prefix'); + expect(values).not.toContain('suffix'); + }); + + it('reads a nested template inside a hole', () => { + expect(() => scan('({ ctx }) => `${`${ctx.a}`}`')).not.toThrow(); + expect(rejects('({ ctx }) => `${`${nested}`}`').message).toContain('`nested`'); + }); + + it('does not mistake division for a regex', () => { + expect(() => scan('({ ctx }) => (ctx.a + ctx.b) / 2')).not.toThrow(); + }); +}); + +/** + * Known misses, not bugs: the scan collects bindings flat rather than + * per-scope and treats a name before `:` as a label/key rather than a + * reference (see the module doc). Both directions only ever *accept* code + * that closes over something real — they never reject code that would have + * worked, which is the safe side to be wrong on. Pinned here so a future + * tightening of the scan is a deliberate choice, not an accidental one. + */ +describe('known accept-biased misses (documented, not fixed)', () => { + it('does not resolve the truthy arm of a ternary, so a free name there slips through', () => { + // `freeVar` sits directly before the ternary`s `:` and reads the same + // as a label, so the scan skips it — even though it is a real, + // undeclared reference here. + expect(() => scan('({ event }) => event.ok ? freeVar : event.seq')).not.toThrow(); + }); + + it('over-binds a destructuring rename`s source key', () => { + // `{ event: renamed }` binds only `renamed` — `event` is the property + // being read off the parameter, not a local. The scan collects every + // name in a pattern as bound, so it treats `event` as available too, + // and a bare reference to it below is not caught even though it would + // be a ReferenceError at runtime. + expect(() => + scan('({ event: renamed }) => { return renamed.x + event; }'), + ).not.toThrow(); + }); +}); + +describe('what the scan reports back', () => { + it('lists what the source binds and what it reads', () => { + const { bound, references } = scan('({ event, ctx }) => { const n = ctx.n; return event.seq + n; }'); + + expect([...bound].sort()).toEqual(['ctx', 'event', 'n']); + expect(references).toEqual(['event', 'ctx', 'n']); + }); +}); diff --git a/src/puter-js/src/modules/events/lib/handlerSource.js b/src/puter-js/src/modules/events/lib/handlerSource.js new file mode 100644 index 000000000..ba08900b2 --- /dev/null +++ b/src/puter-js/src/modules/events/lib/handlerSource.js @@ -0,0 +1,151 @@ +import { PuterJSError } from '../../../lib/PuterJSError.js'; +import { scanHandlerSource } from './freeVariables.js'; + +/** + * Turning what a developer wrote into what gets deployed. + * + * A handler is not called where it is written — it is serialized, stored, and + * run later in the app's events worker. So the three accepted forms all reduce + * to one string, that string is scanned for anything it cannot carry with it, + * and its hash goes along so the server can tell whether the code a + * subscription was written against is still what is published. + */ + +/** Hard cap on a subscription's serialized `context`, matching the column. */ +export const CONTEXT_MAX_BYTES = 4096; + +const invalidHandler = (message) => + new PuterJSError(message, 'events_handler_invalid'); + +const contextTooLarge = () => + new PuterJSError( + `Subscription context may not exceed ${CONTEXT_MAX_BYTES} bytes`, + 'events_context_too_large', + ); + +/** + * Hashing is `crypto.subtle`, which an insecure browser origin does not + * provide. Failing loudly beats binding a subscription to whatever happens to + * be published under the name. + */ +const hashUnavailable = () => + new PuterJSError( + 'This environment provides no `crypto.subtle`, so an inline handler cannot be ' + + 'hashed. Publish it with `puter.events.handlers.publish()` and subscribe with ' + + '`handlerName` instead.', + 'events_handler_hash_unavailable', + ); + +const encoder = new TextEncoder(); + +/** Bytes a string takes on the wire, which is what every cap is measured in. */ +export const byteLength = (text) => encoder.encode(text).length; + +/** + * SHA-256 of the source, hex, matching what the server stores. Async because + * `crypto.subtle` is, and it is the only digest all three runtimes share. + * + * @param {string} source + * @returns {Promise} + */ +export const hashSource = async (source) => { + const subtle = globalThis.crypto?.subtle; + if ( ! subtle ) throw hashUnavailable(); + const digest = await subtle.digest('SHA-256', encoder.encode(source)); + return [...new Uint8Array(digest)] + .map(byte => byte.toString(16).padStart(2, '0')) + .join(''); +}; + +/** + * The source of a handler given as a function or a source string. A + * `{ file }` form is read separately, because reading is asynchronous and + * everything else here is not. + * + * @param {unknown} handler + * @returns {string | null} `null` when the handler is a `{ file }` reference. + */ +export const sourceOf = (handler) => { + if ( typeof handler === 'function' ) return Function.prototype.toString.call(handler); + if ( typeof handler === 'string' ) { + if ( handler.trim().length === 0 ) + throw invalidHandler('A handler source string may not be empty'); + return handler; + } + if ( handler && typeof handler === 'object' && 'file' in handler ) return null; + throw invalidHandler( + 'A handler must be a function, a source string, or `{ file: }`', + ); +}; + +/** + * Resolve a handler to its source, reading a `{ file }` reference through the + * caller's own filesystem. + * + * File references resolve **at this call**, not at delivery: what is deployed + * is the bytes as they were when the handler was published or subscribed, so + * editing the file afterwards changes nothing until it is published again. + * + * @param {import('../../../index.js').Puter} puter + * @param {unknown} handler + * @returns {Promise} + */ +export const resolveSource = async (puter, handler) => { + const inline = sourceOf(handler); + if ( inline !== null ) return inline; + + const path = /** @type {{ file: unknown }} */ (handler).file; + if ( typeof path !== 'string' || path.trim().length === 0 ) + throw invalidHandler('`file` must be a non-empty path'); + + const blob = await puter.fs.read(path); + const source = typeof blob === 'string' ? blob : await blob.text(); + if ( source.trim().length === 0 ) + throw invalidHandler(`\`${path}\` is empty`); + return source; +}; + +/** + * Everything the wire needs about a handler: its source, its hash, and the + * guarantee that it names nothing it cannot carry. + * + * @param {import('../../../index.js').Puter} puter + * @param {unknown} handler + * @returns {Promise<{ source: string, hash: string }>} + */ +export const prepareHandler = async (puter, handler) => { + const source = await resolveSource(puter, handler); + scanHandlerSource(source); + return { source, hash: await hashSource(source) }; +}; + +/** + * The `context` a subscription carries, serialized and checked against the cap + * before the request rather than after it. + * + * Evaluated **now**: `ctx` is a snapshot of these values as they are at + * subscribe time, and it never changes again for the life of the subscription. + * + * @param {unknown} context + * @returns {string | undefined} + */ +export const serializeContext = (context) => { + if ( context === undefined || context === null ) return undefined; + + let json; + try { + json = JSON.stringify(context); + } catch { + throw new PuterJSError( + 'context must be JSON-serializable', + 'events_context_invalid', + ); + } + if ( json === undefined ) + throw new PuterJSError( + 'context must be JSON-serializable', + 'events_context_invalid', + ); + if ( byteLength(json) > CONTEXT_MAX_BYTES ) throw contextTooLarge(); + return json; +}; diff --git a/src/puter-js/src/modules/events/lib/handlers.js b/src/puter-js/src/modules/events/lib/handlers.js new file mode 100644 index 000000000..f61d5369c --- /dev/null +++ b/src/puter-js/src/modules/events/lib/handlers.js @@ -0,0 +1,172 @@ +import { PuterJSError } from '../../../lib/PuterJSError.js'; +import { request } from './api.js'; +import { prepareHandler } from './handlerSource.js'; + +/** @typedef {import('../types.js').PublishedHandler} PublishedHandler */ +/** @typedef {import('../types.js').HandlerSummary} HandlerSummary */ +/** @typedef {import('../types.js').HandlerOptions} HandlerOptions */ +/** @typedef {import('../types.js').HandlerPublication} HandlerPublication */ + +/** + * `puter.events.handlers` — the named functions an app deploys once and its + * users' subscriptions bind to. + * + * Publishing is a developer operation: an app token publishes into its own app, + * and a plain session has to name an app it owns. Nothing here triggers a + * handler — a name is a label for deployed code, and it runs only when a + * subscription bound to it has a delivery. + * + * Two build steps publishing different source under one name is a race with no + * right winner, so this sends the hash it last saw published (`ifHash`) and + * lets the server refuse a publish whose base has moved. `replace: true` is how + * a caller says it means to take the name regardless. + */ + +/** One name in one app. The same name means different code in another. */ +const baseKey = (appUid, name) => `${appUid ?? ''}|${name}`; + +const invalidName = () => + new PuterJSError( + 'A handler name must be a non-empty string', + 'events_handler_name_invalid', + ); + +export class EventHandlers { + /** @param {import('../index.js').EventsModule} module */ + constructor (module) { + /** @internal */ + this.module = module; + /** + * @internal The hash last seen published, keyed by app and name — the + * base a publish claims it is updating. Empty until this client has + * published or listed, which is what makes a first publish + * create-or-idempotent. Keyed by app as well as name because one + * name means different code in two apps. + * @type {Map} + */ + this.known = new Map(); + + for ( const name of ['publish', 'publishAll', 'list', 'remove'] ) { + this[name] = this[name].bind(this); + } + } + + /** + * Publishes one named handler. + * + * @param {string} name The name subscriptions bind to. + * @param {Function | string | { file: string }} handler The handler: a + * function (serialized with `toString()`), its source, or a path to read + * it from. A file resolves now, not at delivery. + * @param {HandlerOptions} [options] + * @returns {Promise} + */ + async publish (name, handler, options = {}) { + const [published] = await this.#send( + [{ name, handler, replace: options.replace }], + options.appUid, + '/events/handlers/publish', + ); + return published; + } + + /** + * Publishes a set of handlers in one call — what a build step has. Items + * are taken in order, and one the server refuses stops the pass, so a + * deploy never reports success over a half-published set. + * + * @param {HandlerPublication[]} handlers + * @param {HandlerOptions} [options] + * @returns {Promise} + */ + async publishAll (handlers, options = {}) { + if ( ! Array.isArray(handlers) || handlers.length === 0 ) { + throw new PuterJSError( + '`handlers` must be a non-empty array', + 'invalid_request', + ); + } + return this.#send(handlers, options.appUid, '/events/handlers/publishAll'); + } + + /** + * What this app has published: names, source hashes, and how many + * subscriptions each is carrying. Never the source. + * + * @param {HandlerOptions} [options] + * @returns {Promise} + */ + async list (options = {}) { + const response = await request( + this.module.puter, + '/events/handlers/list', + undefined, + options.appUid ? { appUid: options.appUid } : undefined, + ); + const handlers = /** @type {HandlerSummary[]} */ (response.handlers ?? []); + for ( const handler of handlers ) + this.known.set(baseKey(options.appUid, handler.name), handler.hash); + return handlers; + } + + /** + * Removes a name. With nothing bound to it the handler simply goes; with + * subscriptions on it they are **suspended**, not deleted, and publishing + * the name again resumes them. + * + * @param {string} name + * @param {HandlerOptions} [options] + * @returns {Promise<{ name: string, removed: boolean, suspended: number }>} + */ + async remove (name, options = {}) { + if ( typeof name !== 'string' || name.trim().length === 0 ) throw invalidName(); + const removed = /** @type {{ name: string, removed: boolean, suspended: number }} */ ( + await request(this.module.puter, '/events/handlers/remove', { + name, + ...(options.appUid ? { appUid: options.appUid } : {}), + }) + ); + this.known.delete(baseKey(options.appUid, name)); + return removed; + } + + /** + * @internal Serialize, scan and send one or more publications, then record + * what is now published so the next publish can name its base. + * @param {HandlerPublication[]} items + * @param {string | undefined} appUid + * @param {string} route + * @returns {Promise} + */ + async #send (items, appUid, route) { + const handlers = []; + for ( const item of items ) { + const name = item?.name; + if ( typeof name !== 'string' || name.trim().length === 0 ) throw invalidName(); + + const { source } = await prepareHandler(this.module.puter, item.handler); + const ifHash = this.known.get(baseKey(appUid, name)); + handlers.push({ + name, + source, + ...(item.replace === true ? { replace: true } : {}), + ...(ifHash && item.replace !== true ? { ifHash } : {}), + }); + } + + const body = { + ...(appUid ? { appUid } : {}), + ...(handlers.length === 1 && route.endsWith('/publish') + ? handlers[0] + : { handlers }), + }; + + const response = await request(this.module.puter, route, body); + const published = /** @type {PublishedHandler[]} */ ( + Array.isArray(response.handlers) ? response.handlers : [response] + ); + for ( const handler of published ) + this.known.set(baseKey(appUid, handler.name), handler.hash); + return published; + } +} diff --git a/src/puter-js/src/modules/events/lib/subscription.js b/src/puter-js/src/modules/events/lib/subscription.js index dd115908f..81f4dbdc6 100644 --- a/src/puter-js/src/modules/events/lib/subscription.js +++ b/src/puter-js/src/modules/events/lib/subscription.js @@ -85,11 +85,16 @@ export class EventSubscription { /** * @internal * @param {PuterEvent | PuterKvEvent | EventGapMarker} event + * @param {Record} [ctx] The subscription's stored + * context, which the handler must not be able to mutate: it is one + * snapshot shared across every delivery. * @returns {void} */ - deliver (event) { + deliver (event, ctx) { try { - const result = this.handler({ event }); + const result = this.handler( + ctx === undefined ? { event } : { event, ctx: Object.freeze(ctx) }, + ); if ( result instanceof Promise ) { result.catch(reportHandlerError); } diff --git a/src/puter-js/src/modules/events/lib/subscription.test.js b/src/puter-js/src/modules/events/lib/subscription.test.js new file mode 100644 index 000000000..3c5e7a917 --- /dev/null +++ b/src/puter-js/src/modules/events/lib/subscription.test.js @@ -0,0 +1,49 @@ +import { describe, expect, it, vi } from 'vitest'; +import { EventSubscription } from './subscription.js'; + +/** + * `context` is one snapshot shared across every delivery (R2-16): a handler + * that could mutate it would have every later delivery see the mutation, on + * every subscriber sharing that context. `deliver()` is where a value crosses + * from "stored" to "handed to the developer's code", so it is where the + * freeze has to happen. + */ + +const fakeChannel = { remove: vi.fn() }; + +describe('what a delivery hands the handler', () => { + it('freezes ctx before the handler ever sees it', () => { + const handler = vi.fn(); + const sub = new EventSubscription(fakeChannel, 'fs:~/Documents', handler); + + sub.deliver({ id: 'e1', op: 'write' }, { url: 'https://ingest.example' }); + + expect(handler).toHaveBeenCalledTimes(1); + const [{ ctx }] = handler.mock.calls[0]; + expect(Object.isFrozen(ctx)).toBe(true); + expect(ctx).toEqual({ url: 'https://ingest.example' }); + }); + + it('omits ctx entirely for a subscription that carries none', () => { + const handler = vi.fn(); + const sub = new EventSubscription(fakeChannel, 'fs:~/Documents', handler); + + sub.deliver({ id: 'e1', op: 'write' }); + + expect(handler).toHaveBeenCalledWith({ event: { id: 'e1', op: 'write' } }); + expect('ctx' in handler.mock.calls[0][0]).toBe(false); + }); + + it('does not let the handler write back into the shared context', () => { + const handler = vi.fn((arg) => { + expect(() => { + arg.ctx.url = 'https://tampered.example'; + }).toThrow(); + }); + const sub = new EventSubscription(fakeChannel, 'fs:~/Documents', handler); + + sub.deliver({ id: 'e1', op: 'write' }, { url: 'https://ingest.example' }); + + expect(handler).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/puter-js/src/modules/events/lib/tokenize.js b/src/puter-js/src/modules/events/lib/tokenize.js new file mode 100644 index 000000000..41e1d55b4 --- /dev/null +++ b/src/puter-js/src/modules/events/lib/tokenize.js @@ -0,0 +1,160 @@ +// A tokenizer good enough to tell an identifier *reference* from everything +// that merely looks like one. It is not a parser: it produces a flat token +// stream with strings, comments and regex literals removed, and template +// literals reduced to the tokens inside their `${}` holes. +// +// This exists because the handler scan has to run in the browser, in node and +// in a worker isolate with no parser available and no dependency to add for +// one. Everything it cannot decide, it decides in the direction that produces a +// clear error rather than a silent misreading. + +/** One token. Strings, comments and regex bodies never reach here. */ +/** @typedef {{ type: 'name' | 'num' | 'punct', value: string }} Token */ + +const WHITESPACE = /\s/; +const IDENT_START = /[A-Za-z_$\u00A0-\uFFFF]/; +const IDENT_PART = /[A-Za-z0-9_$\u00A0-\uFFFF]/; +const DIGIT = /[0-9]/; + +const NUMBER = /^(?:0[xX][0-9a-fA-F_]+|0[bB][01_]+|0[oO][0-7_]+|(?:[0-9][0-9_]*)?\.?[0-9][0-9_]*(?:[eE][+-]?[0-9]+)?|[0-9][0-9_]*\.)n?/; + +// Longest first, so `===` is never read as `==` followed by `=`. +const PUNCTUATORS = [ + '>>>=', '...', '===', '!==', '**=', '<<=', '>>=', '>>>', '&&=', '||=', '??=', + '=>', '==', '!=', '<=', '>=', '&&', '||', '??', '?.', '++', '--', + '+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=', '**', '<<', '>>', +]; + +/** + * After these, a `/` opens a regex rather than dividing. `)` and `]` are + * deliberately absent — `(a + b) / 2` is far more common than a regex there — + * while `}` is present, because reading a regex as division would tokenize its + * body and invent identifiers that were never in the code. + */ +const REGEX_AFTER_KEYWORD = new Set([ + 'return', 'typeof', 'instanceof', 'in', 'of', 'new', 'delete', 'void', + 'case', 'do', 'else', 'yield', 'await', 'throw', +]); + +const NO_REGEX_AFTER = new Set([')', ']', '++', '--']); + +/** + * Splits source into tokens. + * + * @param {string} source + * @returns {Token[]} + */ +export const tokenize = (source) => { + /** @type {Token[]} */ + const tokens = []; + /** Braces that are `${` holes, so `}` can hand the template back. */ + const braces = []; + let inTemplate = false; + let i = 0; + + const push = (type, value) => tokens.push({ type, value }); + const previous = () => tokens[tokens.length - 1]; + + const regexAllowed = () => { + const prev = previous(); + if ( ! prev ) return true; + if ( prev.type === 'num' ) return false; + if ( prev.type === 'name' ) return REGEX_AFTER_KEYWORD.has(prev.value); + return ! NO_REGEX_AFTER.has(prev.value); + }; + + /** Walk to the end of a quoted string, honouring escapes. */ + const skipString = (quote) => { + i++; + while ( i < source.length ) { + if ( source[i] === '\\' ) { i += 2; continue; } + if ( source[i] === quote ) { i++; return; } + i++; + } + }; + + /** Walk to the end of a regex literal, including its character classes. */ + const skipRegex = () => { + i++; + let inClass = false; + while ( i < source.length ) { + const ch = source[i]; + if ( ch === '\\' ) { i += 2; continue; } + if ( ch === '\n' ) return; + if ( ch === '[' ) inClass = true; + else if ( ch === ']' ) inClass = false; + else if ( ch === '/' && ! inClass ) { + i++; + while ( i < source.length && IDENT_PART.test(source[i]) ) i++; + return; + } + i++; + } + }; + + while ( i < source.length ) { + const ch = source[i]; + + if ( inTemplate ) { + if ( ch === '\\' ) { i += 2; continue; } + if ( ch === '`' ) { inTemplate = false; i++; continue; } + if ( ch === '$' && source[i + 1] === '{' ) { + // The hole is code, and code is what this is here to read. + braces.push('template'); + inTemplate = false; + i += 2; + continue; + } + i++; + continue; + } + + if ( WHITESPACE.test(ch) ) { i++; continue; } + + if ( ch === '/' && source[i + 1] === '/' ) { + while ( i < source.length && source[i] !== '\n' ) i++; + continue; + } + if ( ch === '/' && source[i + 1] === '*' ) { + const end = source.indexOf('*/', i + 2); + i = end === -1 ? source.length : end + 2; + continue; + } + if ( ch === '/' && regexAllowed() ) { skipRegex(); continue; } + + if ( ch === '"' || ch === "'" ) { skipString(ch); continue; } + if ( ch === '`' ) { inTemplate = true; i++; continue; } + + if ( ch === '{' ) { braces.push('brace'); push('punct', '{'); i++; continue; } + if ( ch === '}' ) { + if ( braces.pop() === 'template' ) { inTemplate = true; i++; continue; } + push('punct', '}'); + i++; + continue; + } + + if ( DIGIT.test(ch) || (ch === '.' && DIGIT.test(source[i + 1] ?? '')) ) { + const match = NUMBER.exec(source.slice(i)); + const text = match ? match[0] : ch; + push('num', text); + i += text.length; + continue; + } + + if ( IDENT_START.test(ch) ) { + let end = i + 1; + while ( end < source.length && IDENT_PART.test(source[end]) ) end++; + push('name', source.slice(i, end)); + i = end; + continue; + } + + const punct = PUNCTUATORS.find(candidate => source.startsWith(candidate, i)); + if ( punct ) { push('punct', punct); i += punct.length; continue; } + + push('punct', ch); + i++; + } + + return tokens; +}; diff --git a/src/puter-js/src/modules/events/list.js b/src/puter-js/src/modules/events/list.js new file mode 100644 index 000000000..c6a558934 --- /dev/null +++ b/src/puter-js/src/modules/events/list.js @@ -0,0 +1,75 @@ +import { fetchAllPages, iteratePages } from '../../lib/pagination.js'; +import { PuterJSError } from '../../lib/PuterJSError.js'; +import { request } from './lib/api.js'; + +/** @typedef {import('./types.js').PersistentSubscription} PersistentSubscription */ +/** @typedef {import('../../lib/types.js').ListPage} SubscriptionPage */ + +/** + * @overload + * @param {import('../../lib/types.js').ListStreamOptions} options + * @returns {AsyncIterableIterator} + */ +/** + * @overload + * @param {import('../../lib/types.js').ListPaginationOptions & ({ cursor: string | null } | { includeTotal: true })} options + * @returns {Promise} + */ +/** + * @overload + * @param {{ limit?: number }} [options] + * @returns {Promise} + */ +/** + * Lists the persistent subscriptions this caller holds, page by page under the + * hood, resolving to a plain array. Passing any pagination param + * (`cursor`/`includeTotal`) switches to a single-request page envelope, and + * `stream: true` returns an async iterator of page envelopes. + * + * An app sees only the subscriptions it created. A session acting for the + * account sees them all, including ones left behind by an app that is gone — + * which is what makes the account the place a stray subscription is revoked + * from. `context` values are never returned; a row reports its key names and a + * hash instead. + * + * @this {import('./index.js').EventsModule} + * @param {...unknown} args + * @returns {Promise | Promise | AsyncIterableIterator} + */ +export function list (...args) { + const { puter } = this; + const opts = /** @type {Record} */ ( + typeof args[0] === 'object' && args[0] !== null ? args[0] : {} + ); + const { limit, cursor, includeTotal, stream } = opts; + const hasCursor = Object.prototype.hasOwnProperty.call(opts, 'cursor'); + + const fetchPage = pageParams => + request(puter, '/events/subscriptions', undefined, { + ...(limit !== undefined ? { limit } : {}), + ...(pageParams.cursor ? { cursor: pageParams.cursor } : {}), + ...(pageParams.includeTotal ? { includeTotal: true } : {}), + }); + + if ( stream === true ) { + return iteratePages(fetchPage, { + cursor: /** @type {string | null | undefined} */ (cursor), + includeTotal: includeTotal === true, + }); + } + + if ( hasCursor || includeTotal !== undefined ) { + if ( includeTotal !== undefined && typeof includeTotal !== 'boolean' ) { + throw new PuterJSError( + '`includeTotal` must be a boolean', + 'invalid_request', + ); + } + return fetchPage({ + cursor: /** @type {string | null} */ (cursor ?? null), + includeTotal: includeTotal === true, + }); + } + + return fetchAllPages(fetchPage); +} diff --git a/src/puter-js/src/modules/events/onPersistent.js b/src/puter-js/src/modules/events/onPersistent.js new file mode 100644 index 000000000..7b91bb95e --- /dev/null +++ b/src/puter-js/src/modules/events/onPersistent.js @@ -0,0 +1,62 @@ +import { PuterJSError } from '../../lib/PuterJSError.js'; +import { request } from './lib/api.js'; +import { prepareHandler, serializeContext } from './lib/handlerSource.js'; +import { assertSubject } from './lib/validate.js'; + +/** @typedef {import('./types.js').OnPersistentOptions} OnPersistentOptions */ +/** @typedef {import('./types.js').PersistentSubscription} PersistentSubscription */ + +/** + * Subscribes to a subject with a subscription that outlives this connection. + * + * Unlike `onLocal()`, nothing about this lives in the page: the subscription is + * stored against the account, keeps matching while the app is closed, and is + * ended by `puter.events.unsubscribe()` rather than by navigating away. What + * runs is the app's published handler, named by `handlerName`. + * + * `context` is evaluated **here, now** — serialized once and delivered to every + * invocation as a frozen `ctx`. It never re-evaluates, so a value read from the + * environment is the value that subscription carries forever. + * + * @this {import('./index.js').EventsModule} + * @param {OnPersistentOptions} options + * @returns {Promise} + */ +export async function onPersistent (options = {}) { + const { puter } = this; + assertSubject(options?.subject); + + const { handler, handlerName } = options; + // An inline handler is source the server has to match against something it + // already has, and a name is the only thing it can match against. + if ( handler !== undefined && handler !== null && ! handlerName ) { + throw new PuterJSError( + 'An inline `handler` needs a `handlerName` to publish it under', + 'events_handler_name_required', + ); + } + + const inline = handler === undefined || handler === null + ? null + : await prepareHandler(puter, handler); + + const body = { + subject: options.subject, + ...(options.delivery ? { delivery: options.delivery } : {}), + ...(options.targets ? { targets: options.targets } : {}), + ...(handlerName ? { handlerName } : {}), + ...(inline ? { handlerHash: inline.hash } : {}), + ...(options.expiresAt !== undefined && options.expiresAt !== null + ? { expiresAt: options.expiresAt } + : {}), + }; + + // Serialized only to check it against the cap before the round trip; the + // request carries the value, which the server stores the same way. + if ( serializeContext(options.context) !== undefined ) + body.context = options.context; + + return /** @type {PersistentSubscription} */ ( + await request(puter, '/events/subscribe', body) + ); +} diff --git a/src/puter-js/src/modules/events/persistent.test.js b/src/puter-js/src/modules/events/persistent.test.js new file mode 100644 index 000000000..42f1b1754 --- /dev/null +++ b/src/puter-js/src/modules/events/persistent.test.js @@ -0,0 +1,379 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +// Every persistent verb goes through the one HTTP helper, so mocking it needs +// no server and still exercises the real request bodies. +const mockRequest = vi.fn(); +vi.mock('./lib/api.js', () => ({ + request: (...args) => mockRequest(...args), +})); + +const { EventHandlers } = await import('./lib/handlers.js'); +const { list } = await import('./list.js'); +const { onPersistent } = await import('./onPersistent.js'); +const { unsubscribe } = await import('./unsubscribe.js'); + +const SUBJECT = 'fs:~/Documents'; +const HANDLER = ({ event, ctx }) => fetch(ctx.url, { body: event.path }); +/** SHA-256 of the serialized `HANDLER`, computed the same way the SDK does. */ +let handlerHash; + +const makeModule = (fsRead) => { + const module = { + puter: { + APIOrigin: 'https://api.test', + fs: { read: fsRead ?? vi.fn() }, + }, + onPersistent, + unsubscribe, + list, + }; + module.handlers = new EventHandlers(module); + return module; +}; + +const bodyOf = (index = 0) => mockRequest.mock.calls[index][2]; +const routeOf = (index = 0) => mockRequest.mock.calls[index][1]; + +const rejects = async (run) => { + try { + await run(); + } catch (error) { + return error; + } + throw new Error('expected a rejection'); +}; + +beforeEach(async () => { + mockRequest.mockReset(); + mockRequest.mockResolvedValue({}); + if ( ! handlerHash ) { + const { hashSource } = await import('./lib/handlerSource.js'); + handlerHash = await hashSource(Function.prototype.toString.call(HANDLER)); + } +}); + +describe('onPersistent', () => { + it('sends the subject and the server`s answer comes straight back', async () => { + const view = { subId: 'app-1#a', subject: SUBJECT }; + mockRequest.mockResolvedValue(view); + + const sub = await makeModule().onPersistent({ subject: SUBJECT }); + + expect(routeOf()).toBe('/events/subscribe'); + expect(bodyOf()).toEqual({ subject: SUBJECT }); + expect(sub).toBe(view); + }); + + it('carries delivery, targets, handlerName and expiry when given', async () => { + await makeModule().onPersistent({ + subject: SUBJECT, + delivery: 'single', + targets: ['worker'], + handlerName: 'ingestUpload', + expiresAt: 4102444800, + }); + + expect(bodyOf()).toEqual({ + subject: SUBJECT, + delivery: 'single', + targets: ['worker'], + handlerName: 'ingestUpload', + expiresAt: 4102444800, + }); + }); + + it('sends an inline handler as a hash, never as source', async () => { + await makeModule().onPersistent({ + subject: SUBJECT, + handlerName: 'ingestUpload', + handler: HANDLER, + }); + + expect(bodyOf().handlerHash).toBe(handlerHash); + expect(bodyOf().source).toBeUndefined(); + expect(JSON.stringify(bodyOf())).not.toContain('fetch('); + }); + + it('refuses an inline handler with no name to publish it under', async () => { + const error = await rejects(() => + makeModule().onPersistent({ subject: SUBJECT, handler: HANDLER }), + ); + + expect(error.code).toBe('events_handler_name_required'); + expect(mockRequest).not.toHaveBeenCalled(); + }); + + it('rejects an inline handler that closes over something', async () => { + const error = await rejects(() => + makeModule().onPersistent({ + subject: SUBJECT, + handlerName: 'ingestUpload', + handler: '({ event }) => fetch(endpoint, { body: event.path })', + }), + ); + + expect(error.code).toBe('events_handler_free_variable'); + expect(error.message).toContain('`endpoint`'); + expect(mockRequest).not.toHaveBeenCalled(); + }); + + it('reads a `{ file }` handler through the caller`s own filesystem', async () => { + const read = vi.fn(async () => ({ + text: async () => '({ event, ctx }) => console.log(event.uid, ctx.url)', + })); + + await makeModule(read).onPersistent({ + subject: SUBJECT, + handlerName: 'ingestUpload', + handler: { file: '~/AppData/handler.js' }, + }); + + expect(read).toHaveBeenCalledWith('~/AppData/handler.js'); + expect(bodyOf().handlerHash).toMatch(/^[0-9a-f]{64}$/); + }); + + it('refuses a subject that is not a non-empty string', async () => { + for ( const subject of [undefined, null, '', ' ', 42] ) { + const error = await rejects(() => + makeModule().onPersistent({ subject }), + ); + expect(error.code).toBe('invalid_subject'); + } + expect(mockRequest).not.toHaveBeenCalled(); + }); + + it('refuses a handler that is none of the three accepted forms', async () => { + const error = await rejects(() => + makeModule().onPersistent({ + subject: SUBJECT, + handlerName: 'x', + handler: 42, + }), + ); + expect(error.code).toBe('events_handler_invalid'); + }); + + describe('context', () => { + it('sends what was passed, evaluated now', async () => { + await makeModule().onPersistent({ + subject: SUBJECT, + context: { url: 'https://ingest.example', retries: 2 }, + }); + + expect(bodyOf().context).toEqual({ + url: 'https://ingest.example', + retries: 2, + }); + }); + + it('refuses one over the cap before the network', async () => { + const error = await rejects(() => + makeModule().onPersistent({ + subject: SUBJECT, + context: { blob: 'x'.repeat(5000) }, + }), + ); + + expect(error.code).toBe('events_context_too_large'); + expect(mockRequest).not.toHaveBeenCalled(); + }); + + it('refuses one that cannot be serialized', async () => { + const cyclic = {}; + cyclic.self = cyclic; + + const error = await rejects(() => + makeModule().onPersistent({ subject: SUBJECT, context: cyclic }), + ); + expect(error.code).toBe('events_context_invalid'); + }); + }); +}); + +describe('unsubscribe', () => { + it('names the subscription to end', async () => { + await makeModule().unsubscribe('app-1#a'); + + expect(routeOf()).toBe('/events/unsubscribe'); + expect(bodyOf()).toEqual({ subId: 'app-1#a' }); + }); + + it('answers an empty id the way the server answers one it cannot find', async () => { + const error = await rejects(() => makeModule().unsubscribe('')); + expect(error.code).toBe('subscription_does_not_exist'); + expect(mockRequest).not.toHaveBeenCalled(); + }); +}); + +describe('list', () => { + it('follows the cursor and resolves to one array', async () => { + mockRequest + .mockResolvedValueOnce({ items: [{ subId: 'a' }], cursor: 'next' }) + .mockResolvedValueOnce({ items: [{ subId: 'b' }] }); + + const rows = await makeModule().list(); + + expect(rows.map(row => row.subId)).toEqual(['a', 'b']); + expect(mockRequest.mock.calls[1][3]).toMatchObject({ cursor: 'next' }); + }); + + it('returns one page envelope when the caller asks for pagination', async () => { + mockRequest.mockResolvedValue({ items: [], cursor: 'c', total: 7 }); + + const page = await makeModule().list({ cursor: null, includeTotal: true }); + + expect(page).toEqual({ items: [], cursor: 'c', total: 7 }); + expect(mockRequest.mock.calls[0][3]).toMatchObject({ includeTotal: true }); + }); + + it('streams page envelopes when asked to', async () => { + mockRequest + .mockResolvedValueOnce({ items: [{ subId: 'a' }], cursor: 'next' }) + .mockResolvedValueOnce({ items: [{ subId: 'b' }] }); + + const pages = []; + for await ( const page of makeModule().list({ stream: true }) ) pages.push(page); + + expect(pages.map(page => page.items[0].subId)).toEqual(['a', 'b']); + }); +}); + +describe('handlers', () => { + it('publishes the serialized source under a name', async () => { + mockRequest.mockResolvedValue({ + name: 'ingestUpload', + hash: handlerHash, + outcome: 'created', + }); + + const published = await makeModule().handlers.publish('ingestUpload', HANDLER); + + expect(routeOf()).toBe('/events/handlers/publish'); + expect(bodyOf()).toEqual({ + name: 'ingestUpload', + source: Function.prototype.toString.call(HANDLER), + }); + expect(published.outcome).toBe('created'); + }); + + it('names the base it is updating once it knows one', async () => { + const module = makeModule(); + mockRequest.mockResolvedValue({ name: 'ingestUpload', hash: 'hash-1' }); + await module.handlers.publish('ingestUpload', HANDLER); + + mockRequest.mockResolvedValue({ name: 'ingestUpload', hash: 'hash-2' }); + await module.handlers.publish('ingestUpload', '({ ctx }) => ctx.url'); + + expect(bodyOf(1).ifHash).toBe('hash-1'); + }); + + it('takes the base from a listing too', async () => { + const module = makeModule(); + mockRequest.mockResolvedValue({ + handlers: [{ name: 'ingestUpload', hash: 'hash-9', subscriptions: 0 }], + }); + await module.handlers.list(); + + mockRequest.mockResolvedValue({ name: 'ingestUpload', hash: 'hash-10' }); + await module.handlers.publish('ingestUpload', HANDLER); + + expect(bodyOf(1).ifHash).toBe('hash-9'); + }); + + it('names no base when the caller means to take the name', async () => { + const module = makeModule(); + mockRequest.mockResolvedValue({ name: 'ingestUpload', hash: 'hash-1' }); + await module.handlers.publish('ingestUpload', HANDLER); + + await module.handlers.publish('ingestUpload', '({ ctx }) => ctx.url', { + replace: true, + }); + + expect(bodyOf(1)).toMatchObject({ replace: true }); + expect(bodyOf(1).ifHash).toBeUndefined(); + }); + + it('publishes a whole set in one call', async () => { + mockRequest.mockResolvedValue({ + handlers: [ + { name: 'a', hash: 'h1' }, + { name: 'b', hash: 'h2' }, + ], + }); + + const published = await makeModule().handlers.publishAll([ + { name: 'a', handler: HANDLER }, + { name: 'b', handler: '({ ctx }) => ctx.url' }, + ]); + + expect(routeOf()).toBe('/events/handlers/publishAll'); + expect(bodyOf().handlers.map(entry => entry.name)).toEqual(['a', 'b']); + expect(published).toHaveLength(2); + }); + + it('rejects a set item that closes over something, before sending anything', async () => { + const error = await rejects(() => + makeModule().handlers.publishAll([ + { name: 'a', handler: HANDLER }, + { name: 'b', handler: '({ event }) => publish(event)' }, + ]), + ); + + expect(error.code).toBe('events_handler_free_variable'); + expect(error.message).toContain('`publish`'); + expect(mockRequest).not.toHaveBeenCalled(); + }); + + it('names an app when the caller is an account session', async () => { + mockRequest.mockResolvedValue({ name: 'a', hash: 'h' }); + await makeModule().handlers.publish('a', HANDLER, { appUid: 'app-7' }); + + expect(bodyOf()).toMatchObject({ appUid: 'app-7' }); + }); + + it('does not carry one app`s base into another`s', async () => { + const module = makeModule(); + mockRequest.mockResolvedValue({ name: 'a', hash: 'hash-1' }); + await module.handlers.publish('a', HANDLER, { appUid: 'app-1' }); + + // The same name in another app is different code, and this publish is + // not an update to anything. + await module.handlers.publish('a', HANDLER, { appUid: 'app-2' }); + expect(bodyOf(1).ifHash).toBeUndefined(); + }); + + it('lists names and hashes, and forgets a name it removes', async () => { + const module = makeModule(); + mockRequest.mockResolvedValue({ + handlers: [{ name: 'a', hash: 'h', updatedAt: 1, subscriptions: 3 }], + }); + + const listed = await module.handlers.list(); + expect(routeOf()).toBe('/events/handlers/list'); + expect(listed).toEqual([ + { name: 'a', hash: 'h', updatedAt: 1, subscriptions: 3 }, + ]); + + mockRequest.mockResolvedValue({ name: 'a', removed: true, suspended: 3 }); + await module.handlers.remove('a'); + expect(routeOf(1)).toBe('/events/handlers/remove'); + expect(module.handlers.known.size).toBe(0); + }); + + it('refuses a name that is not a non-empty string', async () => { + for ( const name of [undefined, '', ' ', 7] ) { + const error = await rejects(() => + makeModule().handlers.publish(name, HANDLER), + ); + expect(error.code).toBe('events_handler_name_invalid'); + } + }); + + it('works when destructured off the module', async () => { + const { publish } = makeModule().handlers; + mockRequest.mockResolvedValue({ name: 'a', hash: 'h' }); + + await publish('a', HANDLER); + expect(routeOf()).toBe('/events/handlers/publish'); + }); +}); diff --git a/src/puter-js/src/modules/events/types.js b/src/puter-js/src/modules/events/types.js index f63ab7bd7..5dae4f9aa 100644 --- a/src/puter-js/src/modules/events/types.js +++ b/src/puter-js/src/modules/events/types.js @@ -63,8 +63,9 @@ * @property {'gap'} op Always `'gap'`. * @property {string} reason Why the delivery was dropped — * `matched_subscription_limit`, `filter_evaluation_limit`, - * `delivery_rate_limit`, or `backlog_overflow` when undelivered events were - * shed to stay inside a backlog cap. + * `delivery_rate_limit`, `backlog_overflow` when undelivered events were + * shed to stay inside a backlog cap, or `suspended_backlog_expired` when a + * suspended subscription held them past its deadline. * @property {number} ts Milliseconds since the epoch. */ @@ -75,6 +76,9 @@ * @typedef {Object} EventDelivery * @property {PuterEvent | PuterKvEvent | EventGapMarker} event The delivered * event, or a gap marker in place of events that were dropped. + * @property {Readonly>} [ctx] The `context` the + * subscription was created with, frozen. Present only for a persistent + * subscription; a session subscription carries none. */ /** @@ -96,3 +100,111 @@ * @property {number} [timeout] How long to wait for the server to answer * `subscribe`, in milliseconds. Default `30000`. */ + +/** + * Options for {@link import('./onPersistent.js').onPersistent}. + * + * @typedef {Object} OnPersistentOptions + * @property {string} subject What to watch — the same grammar `onLocal()` + * takes, e.g. `fs:~/Documents` or `fs:~/inbox/*.json:add`. + * @property {'broadcast' | 'single'} [delivery] `broadcast` (the default) + * delivers to everything listening; `single` delivers to exactly one + * consumer, which must acknowledge, and requires a `handlerName`. + * @property {Array<'socket' | 'worker' | 'push'>} [targets] Transports the + * deliveries may take. Defaults to `['socket', 'worker']`. A `single` + * subscription may not target `push`. + * @property {string} [handlerName] The published handler this subscription + * binds to. Required for `single`. + * @property {Function | string | { file: string }} [handler] The handler + * source this subscription was written against. Sent as a hash, not as + * source: the subscription binds only if it matches what is published under + * `handlerName`, which is also required when this is given. + * @property {Record} [context] Values the handler needs, + * evaluated **now** and delivered as a frozen `ctx` on every invocation. + * Capped at 4 KB serialized. + * @property {number | string} [expiresAt] When the subscription ends by + * itself — unix seconds or an ISO-8601 string, and it has to be in the + * future. + */ + +/** + * A subscription that outlives the connection that made it. + * + * `context` values are deliberately absent: the column holds whatever secret + * the handler needs, and a listing is the one surface an app can call + * repeatedly. What comes back is its shape — which keys are set, and a hash + * that changes when any value does. + * + * @typedef {Object} PersistentSubscription + * @property {string} subId The subscription's id, and what `unsubscribe()` + * names. Stable for the life of the subscription. + * @property {string} subject The subject it was created with. + * @property {EventAnchor} anchor The node it is keyed to. + * @property {string | null} match The pattern events under the anchor are + * matched against, or `null` when the subject named the anchor itself. + * @property {string | null} op The single operation it is limited to, or + * `null` for all of them. + * @property {Array<'socket' | 'worker' | 'push'>} targets Transports its + * deliveries may take. + * @property {'broadcast' | 'single'} delivery Its delivery class. + * @property {string | null} handlerName The handler it is bound to. + * @property {string | null} appUid The app that created it, or `null` for one + * an account session made. + * @property {string[] | null} contextKeys Key names of its stored context, + * never the values, or `null` when it carries none. + * @property {string | null} contextHash Hash of its stored context, so a + * change is visible without the values. + * @property {number} createdAt Unix seconds. + * @property {number | null} expiresAt Unix seconds, or `null` for one with no + * end. + * @property {number | null} suspendedAt When it stopped delivering without + * being removed, or `null` while it is live. + * @property {string | null} suspendedReason Why it stopped — + * `handler_not_found`, `failures`, `no_credit`, or `permission_revoked`. + */ + +/** + * Where a handler operation applies. An app token publishes into its own app + * and needs neither field; an account session has to name an app it owns. + * + * @typedef {Object} HandlerOptions + * @property {boolean} [replace] Take the name whatever is published under it. + * Without this, a publish whose base has moved is refused with + * `events_handler_conflict`. + * @property {string} [appUid] The app to publish into. Required when the + * caller is an account session rather than an app. + */ + +/** + * One item of a `publishAll()` set. + * + * @typedef {Object} HandlerPublication + * @property {string} name The name subscriptions bind to. + * @property {Function | string | { file: string }} handler The handler: a + * function, its source, or a path to read it from. + * @property {boolean} [replace] Take the name whatever is published under it. + */ + +/** + * What a publish reports back. Never the source. + * + * @typedef {Object} PublishedHandler + * @property {string} name + * @property {string} hash SHA-256 of the published source. + * @property {number} updatedAt Unix seconds. + * @property {'created' | 'updated' | 'unchanged'} outcome What the publish + * did. `unchanged` means the same source was already published. + * @property {number} resumed Suspended subscriptions this publish brought + * back into service. + */ + +/** + * One handler as `puter.events.handlers.list()` reports it. + * + * @typedef {Object} HandlerSummary + * @property {string} name + * @property {string} hash SHA-256 of the published source. + * @property {number} updatedAt Unix seconds. + * @property {number} subscriptions How many subscriptions are bound to this + * name, suspended ones included. + */ diff --git a/src/puter-js/src/modules/events/unsubscribe.js b/src/puter-js/src/modules/events/unsubscribe.js new file mode 100644 index 000000000..a8e0f6bc5 --- /dev/null +++ b/src/puter-js/src/modules/events/unsubscribe.js @@ -0,0 +1,23 @@ +import { PuterJSError } from '../../lib/PuterJSError.js'; +import { request } from './lib/api.js'; + +/** + * Ends a persistent subscription. + * + * An id this account does not hold — one already ended, or one another app + * created — reads as absent rather than refused, so the call cannot be used to + * find out which subscriptions exist. + * + * @this {import('./index.js').EventsModule} + * @param {string} subId The `subId` of the subscription to end. + * @returns {Promise} + */ +export async function unsubscribe (subId) { + if ( typeof subId !== 'string' || subId.trim().length === 0 ) { + throw new PuterJSError( + 'No such subscription', + 'subscription_does_not_exist', + ); + } + await request(this.puter, '/events/unsubscribe', { subId }); +} diff --git a/src/puter-js/tests/api/suites/events.suite.ts b/src/puter-js/tests/api/suites/events.suite.ts index 0ceb2fa87..2db58390c 100644 --- a/src/puter-js/tests/api/suites/events.suite.ts +++ b/src/puter-js/tests/api/suites/events.suite.ts @@ -60,12 +60,40 @@ const open = ( timeout: SUBSCRIBE_TIMEOUT_MS, }); +/** A handler that closes over nothing, so the free-variable scan accepts it. */ +const HANDLER = '({ event, ctx }) => { console.log(event.path, ctx.label); }'; +const OTHER_HANDLER = '({ event, ctx }) => { console.log(ctx.label, event.uid); }'; + +/** An app of this account's own, so its handlers are the caller's to publish. */ +const makeApp = async (t: TestContext): Promise => { + const name = unique('events-handlers'); + const app = await t.puter.apps.create(name, `https://example.com/${name}`); + return (app as unknown as { uid: string }).uid; +}; + export default suite('events', { 'exposes onLocal': async (t) => { t.assert.ok(t.puter.events, 'puter.events is registered'); t.assert.equal(typeof t.puter.events.onLocal, 'function'); }, + 'exposes the persistent surface': async (t) => { + for (const method of ['onPersistent', 'unsubscribe', 'list'] as const) { + t.assert.equal( + typeof t.puter.events[method], + 'function', + `puter.events.${method} is a function`, + ); + } + for (const method of ['publish', 'publishAll', 'list', 'remove'] as const) { + t.assert.equal( + typeof t.puter.events.handlers[method], + 'function', + `puter.events.handlers.${method} is a function`, + ); + } + }, + 'rejects a subject that is not a non-empty string': async (t) => { for (const subject of [undefined, null, '', ' ', 42, {}]) { const error = await t.assert.rejects( @@ -319,4 +347,214 @@ export default suite('events', { ); } }, + + // -- Persistent subscriptions ------------------------------------ + + 'refuses an inline handler with no name to publish it under': async (t) => { + const error = await t.assert.rejects(() => + t.puter.events.onPersistent({ + subject: `fs:/${t.env.users.user.username}`, + handler: HANDLER, + }), + ); + t.assert.equal(codeOf(error), 'events_handler_name_required'); + }, + + 'refuses a handler that closes over something it cannot carry': async (t) => { + const error = await t.assert.rejects(() => + t.puter.events.onPersistent({ + subject: `fs:/${t.env.users.user.username}`, + handlerName: 'ingestUpload', + handler: '({ event }) => fetch(ingestUrl, { body: event.path })', + }), + ); + t.assert.equal(codeOf(error), 'events_handler_free_variable'); + t.assert.ok( + (error as Error).message.includes('ingestUrl'), + 'the error names the identifier that could not be resolved', + ); + }, + + 'refuses a context over the cap before the round trip': async (t) => { + const error = await t.assert.rejects(() => + t.puter.events.onPersistent({ + subject: `fs:/${t.env.users.user.username}`, + context: { blob: 'x'.repeat(5000) }, + }), + ); + t.assert.equal(codeOf(error), 'events_context_too_large'); + }, + + 'creates, lists and ends a persistent subscription': async (t) => { + const dir = await makeDir(t, 'events-persistent'); + + const sub = await t.puter.events.onPersistent({ + subject: `fs:${dir}`, + context: { label: 'ingest', token: 'shhh-not-in-a-listing' }, + }); + + try { + t.assert.ok(sub.subId, 'the subscription carries a server id'); + t.assert.equal(sub.subject, `fs:${dir}`); + t.assert.equal(sub.delivery, 'broadcast'); + t.assert.equal(sub.suspendedAt, null); + + const held = await t.puter.events.list(); + const listed = held.find((row) => row.subId === sub.subId); + t.assert.ok(listed, 'the subscription is in the account`s listing'); + // The context is where an API key lives, so a listing reports its + // shape and never its values. + t.assert.deepEqual(listed?.contextKeys, ['label', 'token']); + t.assert.ok( + typeof listed?.contextHash === 'string' && + listed.contextHash.length === 64, + 'the listing carries a content hash of the context', + ); + t.assert.ok( + ! JSON.stringify(listed).includes('shhh-not-in-a-listing'), + 'the listing carries no context values', + ); + } finally { + await t.puter.events.unsubscribe(sub.subId); + } + + const after = await t.puter.events.list(); + t.assert.ok( + ! after.some((row) => row.subId === sub.subId), + 'the subscription is gone once unsubscribed', + ); + }, + + 'answers a listing page when asked for one': async (t) => { + const page = await t.puter.events.list({ cursor: null, includeTotal: true }); + t.assert.ok(Array.isArray(page.items), 'a page carries items'); + t.assert.equal(typeof page.total, 'number', 'a total was requested'); + }, + + 'answers an id it does not hold the way it answers one that is gone': async (t) => { + const error = await t.assert.rejects(() => + t.puter.events.unsubscribe(''), + ); + t.assert.equal(codeOf(error), 'subscription_does_not_exist'); + }, + + 'refuses to bind an inline handler nothing is published for': async (t) => { + const dir = await makeDir(t, 'events-unbound'); + const error = await t.assert.rejects(() => + t.puter.events.onPersistent({ + subject: `fs:${dir}`, + handlerName: unique('missing'), + handler: HANDLER, + }), + ); + t.assert.equal(codeOf(error), 'events_handler_not_found'); + }, + + // -- Handlers ---------------------------------------------------- + + 'publishes, lists and removes a named handler': async (t) => { + const appUid = await makeApp(t); + + const published = await t.puter.events.handlers.publish( + 'ingestUpload', + HANDLER, + { appUid }, + ); + t.assert.equal(published.name, 'ingestUpload'); + t.assert.equal(published.outcome, 'created'); + t.assert.ok( + typeof published.hash === 'string' && published.hash.length === 64, + 'a publish reports the source hash', + ); + + const listed = await t.puter.events.handlers.list({ appUid }); + t.assert.deepEqual( + listed.map((row) => row.name), + ['ingestUpload'], + ); + t.assert.equal(listed[0].subscriptions, 0); + t.assert.ok( + ! JSON.stringify(listed).includes('console.log'), + 'a listing never carries handler source', + ); + + const removed = await t.puter.events.handlers.remove('ingestUpload', { + appUid, + }); + t.assert.equal(removed.removed, true); + t.assert.equal(removed.suspended, 0); + t.assert.deepEqual(await t.puter.events.handlers.list({ appUid }), []); + }, + + 'republishing the same source changes nothing': async (t) => { + const appUid = await makeApp(t); + await t.puter.events.handlers.publish('ingestUpload', HANDLER, { appUid }); + + const again = await t.puter.events.handlers.publish( + 'ingestUpload', + HANDLER, + { appUid }, + ); + t.assert.equal(again.outcome, 'unchanged'); + }, + + 'updates a name it published, and takes one it means to replace': async (t) => { + const appUid = await makeApp(t); + await t.puter.events.handlers.publish('ingestUpload', HANDLER, { appUid }); + + // Having published it, this client knows the base it is updating, so + // the change is accepted rather than read as a racing build step. + const updated = await t.puter.events.handlers.publish( + 'ingestUpload', + OTHER_HANDLER, + { appUid }, + ); + t.assert.equal(updated.outcome, 'updated'); + + const replaced = await t.puter.events.handlers.publish( + 'ingestUpload', + HANDLER, + { appUid, replace: true }, + ); + t.assert.equal(replaced.outcome, 'updated'); + t.assert.equal( + (await t.puter.events.handlers.list({ appUid }))[0].hash, + replaced.hash, + 'the listing reports what the last publish left', + ); + }, + + 'takes a whole set in one call': async (t) => { + const appUid = await makeApp(t); + + const published = await t.puter.events.handlers.publishAll( + [ + { name: 'ingestUpload', handler: HANDLER }, + { name: 'indexDocument', handler: OTHER_HANDLER }, + ], + { appUid }, + ); + + t.assert.deepEqual( + published.map((row) => row.name), + ['ingestUpload', 'indexDocument'], + ); + t.assert.equal((await t.puter.events.handlers.list({ appUid })).length, 2); + }, + + 'refuses to publish into an app this account does not own': async (t) => { + const error = await t.assert.rejects(() => + t.puter.events.handlers.publish('ingestUpload', HANDLER, { + appUid: 'app-00000000-0000-4000-8000-000000000099', + }), + ); + t.assert.equal(codeOf(error), 'events_handler_forbidden'); + }, + + 'refuses to publish without naming an app at all': async (t) => { + const error = await t.assert.rejects(() => + t.puter.events.handlers.publish('ingestUpload', HANDLER), + ); + t.assert.equal(codeOf(error), 'events_handler_app_required'); + }, });