feat: notification scope columns and backfill (PUT-1659) (#3670)

src/backend/clients/database/SqliteDatabaseClient.test.ts

@@ -27,7 +27,7 @@ import { DatabaseClientFactory } from './index.js';
 import { SqliteDatabaseClient } from './SqliteDatabaseClient.js';
 
 /** Highest schema version the migration table can reach. */
-const CURRENT_SCHEMA_VERSION = 67;
+const CURRENT_SCHEMA_VERSION = 68;
 
 /**
  * These suites migrate real files on disk. Idle they finish in well under a
@@ -190,6 +190,183 @@ describe('SqliteDatabaseClient — boot and migrations', { timeout: DISK_MIGRATI
         });
     });
 
+    // 0072 rebuilds `notification` for the scope tuple, and picks up the
+    // user_id index and the delete cascade mysql and postgres already had.
+    describe('0072 notification-scope', () => {
+        // Only the backfill re-runs: the rebuild half is once-only by
+        // construction, but mysql and postgres replay their whole file on
+        // every boot, so these statements have to be safe twice.
+        const BACKFILL = readFileSync(
+            new URL(
+                './migrations/sqlite/0072_notification-scope.sql',
+                import.meta.url,
+            ),
+            'utf8',
+        )
+            .split(/;\s*\n/)
+            .map((s) => s.trim())
+            .filter((s) => /(^|\n)UPDATE /.test(s));
+
+        const seedUser = async (): Promise<number> => {
+            const name = `notif-${Math.random().toString(36).slice(2, 10)}`;
+            await client.write(
+                'INSERT INTO `user` (`username`, `uuid`) VALUES (?, ?)',
+                [name, `${name}-uuid`],
+            );
+            const [row] = await client.read(
+                'SELECT `id` FROM `user` WHERE `username` = ?',
+                [name],
+            );
+            return Number(row.id);
+        };
+
+        const seedLegacy = async (
+            userId: number,
+            value: Record<string, unknown> | string,
+        ): Promise<string> => {
+            const uid = `n-${Math.random().toString(36).slice(2, 12)}`;
+            await client.write(
+                'INSERT INTO `notification` (`uid`, `user_id`, `value`) VALUES (?, ?, ?)',
+                [
+                    uid,
+                    userId,
+                    typeof value === 'string' ? value : JSON.stringify(value),
+                ],
+            );
+            return uid;
+        };
+
+        const scopeOf = async (uid: string) => {
+            const [row] = await client.read(
+                'SELECT `type`, `audience`, `app_uid` FROM `notification` WHERE `uid` = ?',
+                [uid],
+            );
+            return row;
+        };
+
+        const runBackfill = async () => {
+            for (const stmt of BACKFILL) await client.write(stmt);
+        };
+
+        it('carries every backfill statement', () => {
+            expect(BACKFILL).toHaveLength(3);
+        });
+
+        it('classifies the markers legacy rows actually carried', async () => {
+            const userId = await seedUser();
+            const received = await seedLegacy(userId, {
+                source: 'sharing',
+                template: 'file-shared-with-you',
+            });
+            const claimed = await seedLegacy(userId, {
+                source: 'sharing',
+                template: 'file-shared-before-you-joined',
+            });
+            const deployed = await seedLegacy(userId, {
+                source: 'worker',
+                title: 'Successfully deployed https://x.puter.work',
+                template: 'user-requesting-share',
+            });
+            const failed = await seedLegacy(userId, {
+                source: 'worker',
+                title: 'Failed to deploy x! boom',
+                template: 'user-requesting-share',
+            });
+
+            await runBackfill();
+
+            expect(await scopeOf(received)).toEqual({
+                type: 'share.received',
+                audience: 'account',
+                app_uid: null,
+            });
+            expect(await scopeOf(claimed)).toEqual({
+                type: 'share.claimed',
+                audience: 'account',
+                app_uid: null,
+            });
+            // The app is unrecoverable — the payload only ever named a worker.
+            expect(await scopeOf(deployed)).toEqual({
+                type: 'app.worker.deployed',
+                audience: 'developer',
+                app_uid: null,
+            });
+            expect(await scopeOf(failed)).toEqual({
+                type: 'app.worker.deployFailed',
+                audience: 'developer',
+                app_uid: null,
+            });
+        });
+
+        it('leaves anything it does not recognise reading as legacy', async () => {
+            const userId = await seedUser();
+            const unknown = await seedLegacy(userId, {
+                source: 'sharing',
+                template: 'something-else',
+            });
+            const bare = await seedLegacy(userId, { title: 'hi' });
+            const notJson = await seedLegacy(userId, 'plain text');
+
+            await runBackfill();
+
+            for (const uid of [unknown, bare, notJson]) {
+                expect(await scopeOf(uid)).toEqual({
+                    type: '',
+                    audience: 'account',
+                    app_uid: null,
+                });
+            }
+        });
+
+        it('does not reclassify on replay', async () => {
+            const userId = await seedUser();
+            const uid = await seedLegacy(userId, {
+                source: 'sharing',
+                template: 'file-shared-with-you',
+            });
+
+            await runBackfill();
+            // A row already classified as something else must survive a
+            // second pass untouched.
+            await client.write(
+                'UPDATE `notification` SET `audience` = ?, `app_uid` = ? WHERE `uid` = ?',
+                ['app-user', 'app-1234', uid],
+            );
+            await runBackfill();
+
+            expect(await scopeOf(uid)).toEqual({
+                type: 'share.received',
+                audience: 'app-user',
+                app_uid: 'app-1234',
+            });
+        });
+
+        it('indexes user_id and the scope tuple', async () => {
+            const rows = await client.read('PRAGMA index_list(`notification`)');
+            const names = rows.map((r) => String(r.name));
+            expect(names).toContain('idx_notification_user_id');
+            expect(names).toContain('idx_notification_scope');
+        });
+
+        it('retires a deleted user rows', async () => {
+            const [{ foreign_keys: enforcing }] = await client.read(
+                'PRAGMA foreign_keys',
+            );
+            expect(enforcing).toBe(1);
+
+            const userId = await seedUser();
+            const uid = await seedLegacy(userId, { title: 'orphan-to-be' });
+            await client.write('DELETE FROM `user` WHERE `id` = ?', [userId]);
+
+            expect(
+                await client.read(
+                    'SELECT `uid` FROM `notification` WHERE `uid` = ?',
+                    [uid],
+                ),
+            ).toEqual([]);
+        });
+    });
+
     it('leaves an already-migrated database untouched on a second boot', async () => {
         const dir = mkdtempSync(join(tmpdir(), 'puter-sqlite-'));
         const path = join(dir, 'nested', 'puter.sqlite');

src/backend/clients/database/SqliteDatabaseClient.ts

@@ -101,6 +101,7 @@ const AVAILABLE_MIGRATIONS: [number, string[]][] = [
     [64, ['0069_user-block.sql']],
     [65, ['0070_drop-orphaned-default-groups.sql']],
     [66, ['0071_share_issuer_index.sql']],
+    [67, ['0072_notification-scope.sql']],
 ];
 
 export class SqliteDatabaseClient extends AbstractDatabaseClient {

src/backend/clients/database/migrations/mysql/mysql_mig_26.sql

@@ -0,0 +1,80 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- Give `notification` a scope tuple: what the row is (`type`), the app it is
+-- about (`app_uid`, NULL for platform), and who the recipient is in relation
+-- to it (`audience`). Attribution lived inside the `value` JSON as two
+-- free-form markers, `source` and `template`, that could disagree; these
+-- columns replace them and are what a scoped list can be indexed on.
+--
+-- `app_uid` is char(40) to match `apps`.`uid`, which is `app-` plus a uuid.
+--
+-- Idempotent: columns go through `_puter_add_col` (mig_1), the index through
+-- the guarded procedure. There is no per-file applied-state tracking, so every
+-- statement here tolerates a re-run.
+
+CALL _puter_add_col('notification', 'app_uid', '`app_uid` char(40) COLLATE utf8mb4_unicode_ci DEFAULT NULL');
+CALL _puter_add_col('notification', 'audience', '`audience` varchar(16) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''account''');
+CALL _puter_add_col('notification', 'type', '`type` varchar(64) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''''');
+
+DROP PROCEDURE IF EXISTS _puter_add_notification_scope_index;
+DELIMITER //
+CREATE PROCEDURE _puter_add_notification_scope_index()
+BEGIN
+  -- Serves both the scoped list and the unacknowledged count.
+  IF NOT EXISTS (
+    SELECT 1 FROM INFORMATION_SCHEMA.STATISTICS
+    WHERE TABLE_SCHEMA = DATABASE()
+      AND TABLE_NAME = 'notification'
+      AND INDEX_NAME = 'idx_notification_scope'
+  ) THEN
+    ALTER TABLE `notification` ADD INDEX `idx_notification_scope`
+      (`user_id`, `audience`, `app_uid`, `acknowledged`);
+  END IF;
+END //
+DELIMITER ;
+CALL _puter_add_notification_scope_index();
+DROP PROCEDURE IF EXISTS _puter_add_notification_scope_index;
+
+-- Backfill from the markers the payload actually carried. `type = ''` is the
+-- not-yet-classified state, so re-running matches nothing the first pass
+-- claimed, and a row matching no marker keeps the defaults (`''` / 'account')
+-- and reads as legacy.
+
+UPDATE `notification`
+SET `type` = 'share.received', `audience` = 'account', `app_uid` = NULL
+WHERE `type` = ''
+  AND JSON_UNQUOTE(JSON_EXTRACT(`value`, '$.template')) = 'file-shared-with-you';
+
+UPDATE `notification`
+SET `type` = 'share.claimed', `audience` = 'account', `app_uid` = NULL
+WHERE `type` = ''
+  AND JSON_UNQUOTE(JSON_EXTRACT(`value`, '$.template')) = 'file-shared-before-you-joined';
+
+-- Worker rows carried only `source: 'worker'` and a title, so the app they
+-- belong to is unrecoverable and they stay unattributed. The title prefix is
+-- the sole surviving signal of which way the deploy went.
+UPDATE `notification`
+SET `audience` = 'developer',
+    `app_uid` = NULL,
+    `type` = CASE
+      WHEN JSON_UNQUOTE(JSON_EXTRACT(`value`, '$.title')) LIKE 'Successfully deployed %'
+        THEN 'app.worker.deployed'
+      ELSE 'app.worker.deployFailed'
+    END
+WHERE `type` = ''
+  AND JSON_UNQUOTE(JSON_EXTRACT(`value`, '$.source')) = 'worker';

src/backend/clients/database/migrations/postgres/postgres_mig_15.sql

@@ -0,0 +1,55 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- Scope tuple for `notification`. See mysql/mysql_mig_26.sql for the rationale.
+-- `app_uid` is varchar(40) — the width of `apps`.`uid` — and varchar rather
+-- than char for the same reason `uid` is here: blank padding is not what the
+-- column means.
+--
+-- Idempotent via IF NOT EXISTS and the `type = ''` guard on the backfill.
+
+ALTER TABLE notification ADD COLUMN IF NOT EXISTS app_uid varchar(40);
+ALTER TABLE notification ADD COLUMN IF NOT EXISTS audience varchar(16) NOT NULL DEFAULT 'account';
+ALTER TABLE notification ADD COLUMN IF NOT EXISTS type varchar(64) NOT NULL DEFAULT '';
+
+-- Serves both the scoped list and the unacknowledged count.
+CREATE INDEX IF NOT EXISTS idx_notification_scope
+    ON notification (user_id, audience, app_uid, acknowledged);
+
+UPDATE notification
+SET type = 'share.received', audience = 'account', app_uid = NULL
+WHERE type = ''
+  AND value->>'template' = 'file-shared-with-you';
+
+UPDATE notification
+SET type = 'share.claimed', audience = 'account', app_uid = NULL
+WHERE type = ''
+  AND value->>'template' = 'file-shared-before-you-joined';
+
+-- Worker rows carried only `source: 'worker'` and a title, so the app they
+-- belong to is unrecoverable and they stay unattributed. The title prefix is
+-- the sole surviving signal of which way the deploy went.
+UPDATE notification
+SET audience = 'developer',
+    app_uid = NULL,
+    type = CASE
+      WHEN value->>'title' LIKE 'Successfully deployed %'
+        THEN 'app.worker.deployed'
+      ELSE 'app.worker.deployFailed'
+    END
+WHERE type = ''
+  AND value->>'source' = 'worker';

src/backend/clients/database/migrations/sqlite/0072_notification-scope.sql

@@ -0,0 +1,93 @@
+-- Copyright (C) 2024-present Puter Technologies Inc.
+--
+-- This file is part of Puter.
+--
+-- Puter is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as published
+-- by the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- This program is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU Affero General Public License for more details.
+--
+-- You should have received a copy of the GNU Affero General Public License
+-- along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+-- Scope tuple for `notification`. See mysql/mysql_mig_26.sql for the rationale.
+--
+-- Done as the standard 12-step rebuild rather than three ALTERs, because the
+-- same table is missing two things mysql and postgres have had since the v1
+-- schema: an index on `user_id`, which every `listByUserId` scans without, and
+-- the cascade that retires a deleted user's rows. A column add cannot
+-- introduce a foreign key here.
+--
+-- Rows whose user is already gone are dropped rather than carried over: the
+-- cascade this migration adds is what would have removed them.
+
+CREATE TABLE `notification_new` (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT,
+    "user_id" INTEGER NOT NULL,
+    "uid" TEXT NOT NULL UNIQUE,
+    "value" JSON NOT NULL,
+    "acknowledged" INTEGER DEFAULT NULL,
+    "shown" INTEGER DEFAULT NULL,
+    "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "app_uid" TEXT DEFAULT NULL,
+    "audience" TEXT NOT NULL DEFAULT 'account',
+    "type" TEXT NOT NULL DEFAULT '',
+    FOREIGN KEY("user_id") REFERENCES `user` ("id") ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+INSERT INTO `notification_new` (
+    `id`, `user_id`, `uid`, `value`, `acknowledged`, `shown`, `created_at`
+)
+SELECT
+    `id`, `user_id`, `uid`, `value`, `acknowledged`, `shown`, `created_at`
+FROM `notification`
+WHERE `user_id` IN (SELECT `id` FROM `user`);
+
+DROP TABLE `notification`;
+
+ALTER TABLE `notification_new` RENAME TO `notification`;
+
+CREATE INDEX IF NOT EXISTS `idx_notification_user_id`
+    ON `notification` (`user_id`);
+
+-- Serves both the scoped list and the unacknowledged count.
+CREATE INDEX IF NOT EXISTS `idx_notification_scope`
+    ON `notification` (`user_id`, `audience`, `app_uid`, `acknowledged`);
+
+-- Backfill from the markers the payload actually carried. `type = ''` is the
+-- not-yet-classified state, so a re-run matches nothing the first pass
+-- claimed, and a row matching no marker keeps the defaults (`''` / 'account')
+-- and reads as legacy. `json_valid` guards rows whose `value` was stored as a
+-- bare string.
+
+UPDATE `notification`
+SET `type` = 'share.received', `audience` = 'account', `app_uid` = NULL
+WHERE `type` = ''
+  AND json_valid(`value`)
+  AND json_extract(`value`, '$.template') = 'file-shared-with-you';
+
+UPDATE `notification`
+SET `type` = 'share.claimed', `audience` = 'account', `app_uid` = NULL
+WHERE `type` = ''
+  AND json_valid(`value`)
+  AND json_extract(`value`, '$.template') = 'file-shared-before-you-joined';
+
+-- Worker rows carried only `source: 'worker'` and a title, so the app they
+-- belong to is unrecoverable and they stay unattributed. The title prefix is
+-- the sole surviving signal of which way the deploy went.
+UPDATE `notification`
+SET `audience` = 'developer',
+    `app_uid` = NULL,
+    `type` = CASE
+      WHEN json_extract(`value`, '$.title') LIKE 'Successfully deployed %'
+        THEN 'app.worker.deployed'
+      ELSE 'app.worker.deployFailed'
+    END
+WHERE `type` = ''
+  AND json_valid(`value`)
+  AND json_extract(`value`, '$.source') = 'worker';

src/backend/stores/notification/NotificationStore.js

@@ -102,15 +102,27 @@ export class NotificationStore extends PuterStore {
      * the uid to the socket before this insert lands, and the ack / mark-shown
      * round trip comes back keyed on it.
      *
-     * @param {{ userId: number; value: unknown; uid?: string }} args
+     * `type`, `audience` and `appUid` are the scope tuple; the registry in the
+     * notification service is what decides which combinations are legal, and
+     * the empty `type` written by a caller that names none reads as legacy.
+     *
+     * @param {{ userId: number; value: unknown; uid?: string; type?: string;
+     *   audience?: string; appUid?: string | null }} args
      */
-    async create({ userId, value, uid = uuidv4() }) {
+    async create({
+        userId,
+        value,
+        uid = uuidv4(),
+        type = '',
+        audience = 'account',
+        appUid = null,
+    }) {
         if (!userId) throw new Error('create: userId is required');
         const serialized =
             typeof value === 'string' ? value : JSON.stringify(value ?? {});
         await this.clients.db.write(
-            'INSERT INTO `notification` (`uid`, `user_id`, `value`) VALUES (?, ?, ?)',
-            [uid, userId, serialized],
+            'INSERT INTO `notification` (`uid`, `user_id`, `value`, `type`, `audience`, `app_uid`) VALUES (?, ?, ?, ?, ?, ?)',
+            [uid, userId, serialized, type, audience, appUid],
         );
         await this.#invalidateUnack(userId);
         return this.getByUid(uid, { userId });

src/backend/stores/notification/NotificationStore.test.js

@@ -94,6 +94,69 @@ describe('NotificationStore', () => {
         );
     });
 
+    // -- scope tuple ---------------------------------------------------
+
+    it('persists the scope tuple it was given', async () => {
+        const appUid = `app-${uuidv4()}`;
+        const created = await store.create({
+            userId: user.id,
+            value: { title: 'Deploy failed' },
+            type: 'app.worker.deployFailed',
+            audience: 'developer',
+            appUid,
+        });
+
+        expect(created.type).toBe('app.worker.deployFailed');
+        expect(created.audience).toBe('developer');
+        expect(created.app_uid).toBe(appUid);
+
+        const reread = await store.getByUid(created.uid, { userId: user.id });
+        expect(reread.app_uid).toBe(appUid);
+    });
+
+    it('defaults a caller that names no scope to an unattributed account row', async () => {
+        const created = await store.create({
+            userId: user.id,
+            value: { title: 'Legacy' },
+        });
+        expect(created.type).toBe('');
+        expect(created.audience).toBe('account');
+        expect(created.app_uid).toBeNull();
+    });
+
+    it('carries the scope tuple through listing', async () => {
+        const u = await makeUser();
+        const appUid = `app-${uuidv4()}`;
+        await store.create({
+            userId: u.id,
+            value: {},
+            type: 'share.received',
+            audience: 'account',
+        });
+        await store.create({
+            userId: u.id,
+            value: {},
+            type: 'app.events.ended',
+            audience: 'developer',
+            appUid,
+        });
+
+        const byType = Object.fromEntries(
+            (await store.listByUserId(u.id)).map((r) => [
+                r.type,
+                { audience: r.audience, appUid: r.app_uid },
+            ]),
+        );
+        expect(byType['share.received']).toEqual({
+            audience: 'account',
+            appUid: null,
+        });
+        expect(byType['app.events.ended']).toEqual({
+            audience: 'developer',
+            appUid,
+        });
+    });
+
     it('returns null for an unknown uid', async () => {
         expect(await store.getByUid('no-such-notification')).toBeNull();
     });