diff --git a/src/backend/drivers/apps/AppDriver.js b/src/backend/drivers/apps/AppDriver.js index d64e67567..4b09fa607 100644 --- a/src/backend/drivers/apps/AppDriver.js +++ b/src/backend/drivers/apps/AppDriver.js @@ -34,7 +34,7 @@ import { buildHostedBackingDenial, buildHostedSubdomainIndexUrlCandidates, extractPuterHostedSubdomain, - hostedIndexUrlBackingIsUnavailable, + hostedIndexUrlBackingsAreUnavailable, } from '../../util/hostedAppBacking.js'; import { decodeCursor, @@ -119,6 +119,19 @@ const hasIndexUrlUniquenessExemption = (candidates) => { return false; }; +/** Parsed `protocol//hostname[:port]` origin of an index_url, or null. */ +function indexUrlOrigin(indexUrl) { + if (!indexUrl) return null; + try { + const parsed = new URL(indexUrl); + return `${parsed.protocol}//${parsed.hostname}${ + parsed.port ? `:${parsed.port}` : '' + }`; + } catch { + return null; + } +} + /** * Driver exposing the `puter-apps` interface. * @@ -407,20 +420,34 @@ export class AppDriver extends PuterDriver { // Pre-fetch in parallel: // - per-uid stats (already pipelined inside getAppsStats) // - filetype associations as a single IN-list query (was N queries) - const [statsByUid, filetypesByAppId] = await Promise.all([ + // - canonical-index-url resolution and the hosted-backing check, + // each batched across every visible app + const [ + statsByUid, + filetypesByAppId, + canonicalByApp, + hostedUnavailableByApp, + ] = await Promise.all([ this.appStore.getAppsStats(visible.map((a) => a.uid)), this.appStore.getFiletypeAssociationsByIds( visible.map((a) => a.id), ), + this.#resolveCanonicalForIndexUrls(visible), + this.#hostedBackingUnavailableFlags(visible), ]); const items = await Promise.all( - visible.map((app) => - this.#toClient(app, actor, { - ...params, - stats: statsByUid.get(app.uid), - filetypes: filetypesByAppId.get(app.id) ?? [], - }), + visible.map((app, i) => + this.#toClient( + app, + actor, + { ...params, stats: statsByUid.get(app.uid) }, + { + filetypes: filetypesByAppId.get(app.id) ?? [], + canonical: canonicalByApp[i], + hostedBackingUnavailable: hostedUnavailableByApp[i], + }, + ), ), ); if (!paginated) return items; @@ -891,9 +918,9 @@ export class AppDriver extends PuterDriver { // -- Serialization ------------------------------------------------ /** - * Resolve the canonical app row that backs `app.index_url`. - * - * Returns `{ origin, expectedUid, canonicalApp }`: + * Resolve the canonical app row backing each app's `index_url`, batched + * across `apps`. Returns an array aligned with `apps`, each entry `{ + * origin, expectedUid, canonicalApp }`: * * - `origin` — the parsed origin string from `index_url`. * - `expectedUid` — the canonical app uid for that origin (oldest @@ -913,61 +940,94 @@ export class AppDriver extends PuterDriver { * pre-existing data from before the `subdomain_not_owned` check leaks * the victim's index_url. * - * Returns `null` when there's no `index_url` or it doesn't parse. + * An entry is `null` when its app has no `index_url` or it doesn't parse. */ - async #resolveCanonicalForIndexUrl(app) { - if (!app.index_url) return null; - let origin; - try { - const parsed = new URL(app.index_url); - origin = `${parsed.protocol}//${parsed.hostname}${ - parsed.port ? `:${parsed.port}` : '' - }`; - } catch { - return null; + async #resolveCanonicalForIndexUrls(apps) { + const origins = apps.map((app) => indexUrlOrigin(app.index_url)); + const uniqueOrigins = [...new Set(origins.filter((o) => o !== null))]; + + let uidByOrigin = new Map(); + if (uniqueOrigins.length > 0) { + try { + uidByOrigin = + await this.services.auth.appUidsFromOrigins(uniqueOrigins); + } catch { + uidByOrigin = new Map(); + } } - try { - const expectedUid = - await this.services.auth.appUidFromOrigin(origin); - // Avoid a needless DB hit on the self-match common case — - // `app` is already the row we'd be re-fetching. - const canonicalApp = - expectedUid && expectedUid !== app.uid - ? await this.appStore.getByUid(expectedUid) - : app; - return { origin, expectedUid, canonicalApp }; - } catch { - return null; + + // Avoid a needless DB hit on the self-match common case — `app` is + // already the row we'd be re-fetching. + const uidsToFetch = new Set(); + for (let i = 0; i < apps.length; i++) { + if (!origins[i]) continue; + const expectedUid = uidByOrigin.get(origins[i]) ?? null; + if (expectedUid && expectedUid !== apps[i].uid) { + uidsToFetch.add(expectedUid); + } } + + let canonicalAppByUid = new Map(); + if (uidsToFetch.size > 0) { + try { + canonicalAppByUid = await this.appStore.getByUids([ + ...uidsToFetch, + ]); + } catch { + canonicalAppByUid = new Map(); + } + } + + return apps.map((app, i) => { + const origin = origins[i]; + if (!origin) return null; + const expectedUid = uidByOrigin.get(origin) ?? null; + if (!expectedUid) return null; + if (expectedUid === app.uid) { + return { origin, expectedUid, canonicalApp: app }; + } + return { + origin, + expectedUid, + canonicalApp: canonicalAppByUid.get(expectedUid) ?? null, + }; + }); } /** - * Launch-safety check for puter-hosted `index_url`s. See - * `util/hostedAppBacking.ts` — the check lives there because every producer - * of launchable app metadata needs it, not just this driver. + * Launch-safety check for puter-hosted `index_url`s, batched across `apps`. + * See `util/hostedAppBacking.ts` — the check lives there because every + * producer of launchable app metadata needs it, not just this driver. */ - async #hostedIndexUrlBackingIsUnavailable(app) { - return hostedIndexUrlBackingIsUnavailable({ - app, + async #hostedBackingUnavailableFlags(apps) { + return hostedIndexUrlBackingsAreUnavailable({ + apps, subdomainStore: this.stores.subdomain, config: this.config, }); } - async #toClient(app, actor, params = {}) { + async #toClient(app, actor, params = {}, prefetched = {}) { if (!app) return null; - // `select` pre-fetches filetypes for every visible app in one - // batched query and threads them through `params.filetypes` to - // avoid the N+1 in this hot loop. Single-app callers (`read`, - // `create`, `update`) fall back to the per-app query. + // `select` batches these lookups per page and passes them in + // `prefetched`; single-app callers resolve them here. Never read them + // from `params`, which carries the RPC caller's input. const [filetypes, canonicalForIndexUrl, hostedBackingUnavailable] = await Promise.all([ - params.filetypes !== undefined - ? Promise.resolve(params.filetypes) + prefetched.filetypes !== undefined + ? Promise.resolve(prefetched.filetypes) : this.appStore.getFiletypeAssociations(app.id), - this.#resolveCanonicalForIndexUrl(app), - this.#hostedIndexUrlBackingIsUnavailable(app), + prefetched.canonical !== undefined + ? Promise.resolve(prefetched.canonical) + : this.#resolveCanonicalForIndexUrls([app]).then( + (r) => r[0], + ), + prefetched.hostedBackingUnavailable !== undefined + ? Promise.resolve(prefetched.hostedBackingUnavailable) + : this.#hostedBackingUnavailableFlags([app]).then( + (r) => r[0], + ), ]); const createdFromOrigin = diff --git a/src/backend/drivers/apps/AppDriver.test.ts b/src/backend/drivers/apps/AppDriver.test.ts index e339737aa..88871262c 100644 --- a/src/backend/drivers/apps/AppDriver.test.ts +++ b/src/backend/drivers/apps/AppDriver.test.ts @@ -1983,3 +1983,358 @@ describe('AppDriver hosted-subdomain ownership check', () => { ).not.toBe(false); }); }); + +// -- select: batched #toClient query counts -- +// +// Canonical index_url resolution and the hosted-backing check are batched per +// page, so these counts must stay flat as the page grows. + +describe('AppDriver.select query-count regression', () => { + const makeMixedApps = async (count: number) => { + const { actor, userId } = await makeUser(); + for (let i = 0; i < count; i++) { + const kind = i % 3; + if (kind === 0) { + // Live hosted: owns the subdomain it points at. + const sub = uniqueName(`qclive${i}`); + await server.stores.subdomain.create({ userId, subdomain: sub }); + await withActor(actor, () => + driver.create({ + object: { + name: uniqueName(`qc-live-${i}`), + title: 't', + index_url: `https://${sub}.site.puter.localhost/`, + }, + }), + ); + } else if (kind === 1) { + // Dangling hosted: the subdomain is gone by the time we read. + const sub = uniqueName(`qcdang${i}`); + const row = await server.stores.subdomain.create({ + userId, + subdomain: sub, + }); + await withActor(actor, () => + driver.create({ + object: { + name: uniqueName(`qc-dangling-${i}`), + title: 't', + index_url: `https://${sub}.site.puter.localhost/`, + }, + }), + ); + await server.stores.subdomain.deleteByUuid( + String((row as { uuid: string }).uuid), + { userId }, + ); + } else { + // External, with a path — not puter-hosted at all. + await withActor(actor, () => + driver.create({ + object: { + name: uniqueName(`qc-ext-${i}`), + title: 't', + index_url: `${uniqueIndexUrl()}some/path`, + }, + }), + ); + } + } + return actor; + }; + + const countQueriesForSelect = async (actor: Actor) => { + const read = vi.spyOn(server.clients.db, 'read'); + const pread = vi.spyOn(server.clients.db, 'pread'); + try { + const items = (await withActor(actor, () => + driver.select({ predicate: ['user-can-edit'] }), + )) as Array>; + const counts = { indexUrlIn: 0, subdomains: 0, appsUidEq: 0 }; + for (const call of read.mock.calls) { + const sql = call[0] as string; + if (/`index_url`\s+IN\s*\(/i.test(sql)) { + counts.indexUrlIn++; + } else if (sql.includes('`subdomains`')) { + counts.subdomains++; + } else if ( + sql.includes('`apps`') && + /`uid`\s*=\s*\?/.test(sql) + ) { + counts.appsUidEq++; + } + } + return { + counts, + pread: pread.mock.calls.length, + itemCount: items.length, + }; + } finally { + read.mockRestore(); + pread.mockRestore(); + } + }; + + it('keeps index_url / subdomains / per-app-uid query counts constant from 3 apps to 30 apps', async () => { + const actor3 = await makeMixedApps(3); + const actor30 = await makeMixedApps(30); + + const small = await countQueriesForSelect(actor3); + const large = await countQueriesForSelect(actor30); + + expect(small.itemCount).toBe(3); + expect(large.itemCount).toBe(30); + // Sanity check the classifier actually saw the shapes it's counting. + expect(small.counts.indexUrlIn).toBeGreaterThan(0); + expect(small.counts.subdomains).toBeGreaterThan(0); + + expect(large.counts).toEqual(small.counts); + expect(large.pread).toBe(small.pread); + }); +}); + +// -- select / read parity -- + +describe('AppDriver.select / read parity', () => { + const findViaBroadSelect = async ( + actor: Actor, + uid: string, + ): Promise | undefined> => { + let cursor: string | null | undefined = null; + do { + const page = (await withActor(actor, () => + driver.select({ limit: 50, cursor }), + )) as { items: Array>; cursor?: string }; + const found = page.items.find((r) => r.uid === uid); + if (found) return found; + cursor = page.cursor; + } while (cursor); + return undefined; + }; + + it('every select item (minus stats) deep-equals the corresponding read', async () => { + const { actor } = await makeUser(); + const names: string[] = []; + for (let i = 0; i < 4; i++) { + const name = uniqueName(`parity${i}`); + names.push(name); + await withActor(actor, () => + driver.create({ + object: { name, title: 't', index_url: uniqueIndexUrl() }, + }), + ); + } + + const selectResult = (await withActor(actor, () => + driver.select({ predicate: ['user-can-edit'] }), + )) as Array>; + const ours = selectResult.filter((item) => + names.includes(item.name as string), + ); + expect(ours.length).toBe(names.length); + + for (const item of ours) { + const read = await withActor(actor, () => + driver.read({ uid: item.uid as string }), + ); + const { stats: _itemStats, ...itemRest } = item; + const { stats: _readStats, ...readRest } = read; + expect(itemRest).toEqual(readRest); + } + }); + + it('sets created_from_origin on the canonical hosted row and null on a duplicate', async () => { + const { actor, userId } = await makeUser(); + const sub = uniqueName('cfo'); + await server.stores.subdomain.create({ userId, subdomain: sub }); + const url = `https://${sub}.site.puter.localhost/`; + const canonical = await withActor(actor, () => + driver.create({ + object: { + name: uniqueName('cfo-canon'), + title: 't', + index_url: url, + }, + }), + ); + // A duplicate row at the same index_url — `create` would normally + // refuse this; direct insert mirrors the pre-existing-data shape + // the canonical resolver has to cope with. + const dupUid = `app-${uuidv4()}`; + await server.clients.db.write( + 'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)', + [dupUid, uniqueName('cfo-dup'), 'dup', url, userId], + ); + + const result = (await withActor(actor, () => + driver.select({ predicate: ['user-can-edit'] }), + )) as Array>; + const canonItem = result.find((r) => r.uid === canonical.uid); + const dupItem = result.find((r) => r.uid === dupUid); + expect(canonItem?.created_from_origin).toBe( + `https://${sub}.site.puter.localhost`, + ); + expect(dupItem?.created_from_origin).toBeNull(); + }); + + it('gates the canonical-private row: a public duplicate withholds index_url and denies access', async () => { + const ownerA = await makeUser(); + const ownerB = await makeUser(); + + const sharedUrl = uniqueIndexUrl(); + const uidA = `app-${uuidv4()}`; + const uidB = `app-${uuidv4()}`; + await server.clients.db.write( + 'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`, `is_private`) VALUES (?, ?, ?, ?, ?, ?)', + [ + uidA, + uniqueName('priv-a'), + 'Private A', + sharedUrl, + ownerA.userId, + 1, + ], + ); + await server.clients.db.write( + 'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`, `is_private`) VALUES (?, ?, ?, ?, ?, ?)', + [ + uidB, + uniqueName('pub-b'), + 'Public B', + sharedUrl, + ownerB.userId, + 0, + ], + ); + + // B's own owner is not A's owner, so the gate still applies to them. + const result = (await withActor(ownerB.actor, () => + driver.select({ predicate: ['user-can-edit'] }), + )) as Array>; + const bItem = result.find((r) => r.uid === uidB); + expect(bItem).toBeTruthy(); + expect(bItem!.index_url).toBeUndefined(); + expect( + (bItem!.privateAccess as { hasAccess?: boolean }).hasAccess, + ).toBe(false); + }); + + it('dangling hosted app via select: owner keeps index_url, others do not', async () => { + const owner = await makeUser(); + const other = await makeUser(); + const sub = uniqueName('qcdangsel'); + const row = await server.stores.subdomain.create({ + userId: owner.userId, + subdomain: sub, + }); + const created = await withActor(owner.actor, () => + driver.create({ + object: { + name: uniqueName('dangling-sel'), + title: 't', + index_url: `https://${sub}.site.puter.localhost/`, + }, + }), + ); + await server.stores.subdomain.deleteByUuid( + String((row as { uuid: string }).uuid), + { userId: owner.userId }, + ); + + const ownerResult = (await withActor(owner.actor, () => + driver.select({ predicate: ['user-can-edit'] }), + )) as Array>; + const ownerItem = ownerResult.find((r) => r.uid === created.uid); + expect(ownerItem?.index_url).toBe(created.index_url); + expect( + (ownerItem?.privateAccess as { reason?: string } | undefined) + ?.reason, + ).toBe('hosted_backing_unavailable'); + + const otherItem = await findViaBroadSelect( + other.actor, + created.uid as string, + ); + expect(otherItem?.index_url).toBeUndefined(); + expect( + (otherItem?.privateAccess as { hasAccess?: boolean } | undefined) + ?.hasAccess, + ).toBe(false); + }); + + it('a blocked origin resolves created_from_origin to null without select throwing', async () => { + const { actor } = await makeUser(); + const blockedHost = `blocked-${Math.random().toString(36).slice(2, 10)}.test`; + const created = await withActor(actor, () => + driver.create({ + object: { + name: uniqueName('blocked-sel'), + title: 't', + index_url: `https://${blockedHost}/app`, + }, + }), + ); + await server.clients.db.write( + 'INSERT INTO `blocked_app_origins` (`domain`, `include_subdomains`) VALUES (?, ?)', + [blockedHost, 0], + ); + ( + server.services.appOriginBlocklist as { invalidate: () => void } + ).invalidate(); + + const result = (await withActor(actor, () => + driver.select({ predicate: ['user-can-edit'] }), + )) as Array>; + const item = result.find((r) => r.uid === created.uid); + expect(item).toBeTruthy(); + expect(item!.created_from_origin).toBeNull(); + }); +}); + +// -- read: prefetched-shaped params cannot be spoofed -- + +describe('AppDriver.read prefetched-param isolation', () => { + it('ignores caller-supplied hostedBackingUnavailable/canonical/filetypes on a dangling app', async () => { + const owner = await makeUser(); + const attacker = await makeUser(); + const sub = uniqueName('sec-dangling'); + const row = await server.stores.subdomain.create({ + userId: owner.userId, + subdomain: sub, + }); + const created = await withActor(owner.actor, () => + driver.create({ + object: { + name: uniqueName('sec-app'), + title: 't', + index_url: `https://${sub}.site.puter.localhost/`, + filetype_associations: ['.puter'], + }, + }), + ); + await server.stores.subdomain.deleteByUuid( + String((row as { uuid: string }).uuid), + { userId: owner.userId }, + ); + + const result = await withActor(attacker.actor, () => + driver.read({ + uid: created.uid, + params: { + hostedBackingUnavailable: false, + canonical: null, + filetypes: ['x'], + }, + }), + ); + + // The spoofed `hostedBackingUnavailable: false` doesn't suppress the + // real denial, so the dangling app's index_url is still withheld. + expect(result.index_url).toBeUndefined(); + // The spoofed `filetypes` doesn't override the DB-backed list. + expect(result.filetype_associations).toEqual( + expect.arrayContaining(['puter']), + ); + expect(result.filetype_associations).not.toContain('x'); + }); +}); diff --git a/src/backend/drivers/subdomain/SubdomainDriver.test.ts b/src/backend/drivers/subdomain/SubdomainDriver.test.ts index d1db2066b..a5b1d8d16 100644 --- a/src/backend/drivers/subdomain/SubdomainDriver.test.ts +++ b/src/backend/drivers/subdomain/SubdomainDriver.test.ts @@ -1074,3 +1074,48 @@ describe('SubdomainDriver associated_app derivation', () => { expect(read.associated_app).toBeNull(); }); }); + +// -- associated_app derivation at scale -- +// +// ~24 index_url candidates per subdomain: a page this size must be chunked +// to stay under SQLite's bound-parameter limit. + +describe('SubdomainDriver.select at scale', () => { + it('resolves associated_app for 1,400 subdomains without tripping the SQL variable limit', async () => { + const { actor, userId } = await makeUser(); + const prefix = `scale-${Math.random().toString(36).slice(2, 8)}-`; + const count = 1400; + const names: string[] = []; + for (let i = 0; i < count; i++) { + names.push(`${prefix}${i}`); + } + + // Bulk-insert directly — bypasses the driver's per-user quota, the + // same way a bulk-provisioned fleet of sites would exist in prod. + const placeholders = names.map(() => '(?, ?, ?)').join(', '); + const values: unknown[] = []; + for (const name of names) { + values.push(uuidv4(), name, userId); + } + await server.clients.db.write( + `INSERT INTO \`subdomains\` (\`uuid\`, \`subdomain\`, \`user_id\`) VALUES ${placeholders}`, + values, + ); + + const lastSub = names[names.length - 1]!; + const app = await createAppWithIndexUrl( + userId, + `http://${lastSub}.site.puter.localhost/`, + ); + + const result = (await withActor(actor, () => + driver.select({}), + )) as Array>; + + expect(result.length).toBe(count); + const lastItem = result.find((r) => r.subdomain === lastSub); + expect( + (lastItem?.associated_app as { uid: string } | null)?.uid, + ).toBe(app.uid); + }, 30_000); +}); diff --git a/src/backend/drivers/subdomain/SubdomainDriver.ts b/src/backend/drivers/subdomain/SubdomainDriver.ts index 34d6566ea..e9590a70d 100644 --- a/src/backend/drivers/subdomain/SubdomainDriver.ts +++ b/src/backend/drivers/subdomain/SubdomainDriver.ts @@ -188,10 +188,9 @@ export class SubdomainDriver extends PuterDriver { // they have open is not them. // See `util/hostedAppBacking.ts` for the wider rule. // - // Last check before the insert on purpose: `apps.index_url` is - // unindexed, so this scan only runs for a request that would otherwise - // have created the row, and it stays behind the same root_dir gate as - // the existing uniqueness answer. + // Last check before the insert on purpose: it stays behind the same + // root_dir gate as the existing uniqueness answer, so it only runs + // for a request that would otherwise have created the row. const appsHoldingName = (await this.stores.app.listByIndexUrlCandidates( buildHostedSubdomainIndexUrlCandidates(subdomain, this.config), )) as Array>; @@ -737,7 +736,8 @@ export class SubdomainDriver extends PuterDriver { * hosting domains × protocols × paths). Returns a `rowUuid → appId` map. * Rows with no matching app are absent. * - * Runs one batched DB query regardless of input size. + * Goes through the batched `listByIndexUrlCandidates` store lookup rather + * than one query per row, regardless of input size. */ async #deriveAssociatedAppIdByRowUuid( rows: Array>, @@ -782,16 +782,14 @@ export class SubdomainDriver extends PuterDriver { return result; } - const userIds = [...userIdToRowMeta.keys()]; - const userPlaceholders = userIds.map(() => '?').join(', '); const candidateList = [...allCandidates]; - const urlPlaceholders = candidateList.map(() => '?').join(', '); - const matches = (await this.clients.db.read( - `SELECT \`id\`, \`owner_user_id\`, \`index_url\` FROM \`apps\` - WHERE \`owner_user_id\` IN (${userPlaceholders}) - AND \`index_url\` IN (${urlPlaceholders})`, - [...userIds, ...candidateList], + const matches = (await this.stores.app.listByIndexUrlCandidates( + candidateList, )) as Array>; + // Oldest row wins when more than one app matches the same candidate + // for the same owner — the owner-match filter below is what actually + // restricts matches to rows this batch cares about. + matches.sort((a, b) => Number(a.id) - Number(b.id)); for (const m of matches) { const appId = typeof m.id === 'number' ? m.id : Number(m.id); diff --git a/src/backend/services/auth/AuthService.test.ts b/src/backend/services/auth/AuthService.test.ts index 0927e372d..897729258 100644 --- a/src/backend/services/auth/AuthService.test.ts +++ b/src/backend/services/auth/AuthService.test.ts @@ -1811,6 +1811,155 @@ describe('AuthService (integration)', () => { }); }); + describe('appUidsFromOrigins', () => { + it('matches appUidFromOrigin across hosting variants, an unknown external, and a repointed origin', async () => { + const sub = `batch-${Math.random().toString(36).slice(2, 10)}`; + const origins = [ + `https://${sub}.site.puter.localhost`, + `https://${sub}.host.puter.localhost`, + `http://${sub}.app.puter.localhost`, + `https://external-${uuidv4()}.example.com`, + `chrome-extension://${uuidv4()}`, + ]; + + const repointOrigin = `https://repoint-batch-${uuidv4()}.example.com`; + const repointed = await server.stores.app.createFromOrigin( + await authService.appUidFromOrigin(repointOrigin), + authService.canonicalizeOrigin(repointOrigin), + ); + await server.stores.app.update(repointed.id, { + index_url: `https://repoint-new-${uuidv4()}.example.com`, + }); + origins.push(repointOrigin); + + const expected = new Map(); + for (const origin of origins) { + expected.set( + origin, + await authService.appUidFromOrigin(origin), + ); + } + + const batched = await authService.appUidsFromOrigins(origins); + for (const origin of origins) { + expect(batched.get(origin)).toBe(expected.get(origin)); + } + // Every hosting variant of the same subdomain collapses to one uid. + expect(batched.get(origins[0])).toBe(batched.get(origins[1])); + expect(batched.get(origins[0])).toBe(batched.get(origins[2])); + // The repointed origin moved on to a successor uid, not the + // stale row's. + expect(batched.get(repointOrigin)).not.toBe(repointed.uid); + }); + + it('resolves unparseable origins to null instead of throwing', async () => { + const result = await authService.appUidsFromOrigins([ + 'not-a-url', + 'javascript:alert(document.domain)', + ]); + expect(result.get('not-a-url')).toBeNull(); + expect(result.get('javascript:alert(document.domain)')).toBeNull(); + }); + + it('resolves a blocked origin to null without throwing or affecting other origins', async () => { + const blockedHost = `blocked-batch-${uuidv4()}.example.com`; + await server.clients.db.write( + 'INSERT INTO `blocked_app_origins` (`domain`, `include_subdomains`) VALUES (?, ?)', + [blockedHost, 0], + ); + ( + server.services.appOriginBlocklist as { invalidate: () => void } + ).invalidate(); + + const okOrigin = `https://ok-batch-${uuidv4()}.example.com`; + const expectedOk = await authService.appUidFromOrigin(okOrigin); + + const result = await authService.appUidsFromOrigins([ + `https://${blockedHost}`, + okOrigin, + ]); + expect(result.get(`https://${blockedHost}`)).toBeNull(); + expect(result.get(okOrigin)).toBe(expectedOk); + }); + + it('prefers a private row over an older public stub, breaking ties by lowest id, in one batch', async () => { + const user = await makeUser(); + const sub = `batch-priv-${Math.random().toString(36).slice(2, 10)}`; + await server.stores.app.createFromOrigin( + `app-${uuidv4()}`, + `https://${sub}.host.puter.localhost`, + { ownerUserId: user.id }, + ); + const realUid = `app-${uuidv4()}`; + await server.clients.db.write( + 'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`, `is_private`) VALUES (?, ?, ?, ?, ?, ?)', + [ + realUid, + `real-${sub}`, + 'Real app', + `https://${sub}.app.puter.localhost`, + user.id, + 1, + ], + ); + + const origin = `https://${sub}.host.puter.localhost`; + const result = await authService.appUidsFromOrigins([origin]); + expect(result.get(origin)).toBe(realUid); + }); + + it('a private row on one index_url variant wins over a lower-id public row on another, in both paths', async () => { + // Both rows match the same origin's candidate set but live under + // different exact `index_url` strings — the winners reduction has + // to look across every matching variant, not just one. + const base = `https://variant-${uuidv4()}.example.com`; + const publicUid = `app-${uuidv4()}`; + const privateUid = `app-${uuidv4()}`; + await server.clients.db.write( + 'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `is_private`) VALUES (?, ?, ?, ?, ?)', + [publicUid, `pub-${uuidv4()}`, 'Public', `${base}/`, 0], + ); + await server.clients.db.write( + 'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `is_private`) VALUES (?, ?, ?, ?, ?)', + [ + privateUid, + `priv-${uuidv4()}`, + 'Private', + `${base}/index.html`, + 1, + ], + ); + + await expect(authService.appUidFromOrigin(base)).resolves.toBe( + privateUid, + ); + const batched = await authService.appUidsFromOrigins([base]); + expect(batched.get(base)).toBe(privateUid); + }); + + it('does not cross-assign uids between origins that each have their own canonical row', async () => { + const subA = `batch-cross-a-${Math.random().toString(36).slice(2, 8)}`; + const subB = `batch-cross-b-${Math.random().toString(36).slice(2, 8)}`; + const originA = `https://${subA}.site.puter.localhost`; + const originB = `https://${subB}.site.puter.localhost`; + const appA = await server.stores.app.createFromOrigin( + await authService.appUidFromOrigin(originA), + authService.canonicalizeOrigin(originA), + ); + const appB = await server.stores.app.createFromOrigin( + await authService.appUidFromOrigin(originB), + authService.canonicalizeOrigin(originB), + ); + + const result = await authService.appUidsFromOrigins([ + originA, + originB, + ]); + expect(result.get(originA)).toBe(appA.uid); + expect(result.get(originB)).toBe(appB.uid); + }); + }); + describe('subdomainOwnerIdFromOrigin', () => { // Test servers inherit the four hosting domains (production: // puter.site / puter.host / puter.app / puter.dev) from @@ -2937,6 +3086,23 @@ describe('AuthService.appUidFromOrigin — aliased hosts', () => { ); expect(withPort).not.toBe(withoutPort); }); + + it('appUidsFromOrigins collapses an alias group onto one uid, batched', async () => { + const alphaOrigin = 'https://alpha.example.com'; + const betaOrigin = 'https://beta.example.com'; + const ungroupedOrigin = 'https://gamma.example.com'; + + const result = await authService.appUidsFromOrigins([ + alphaOrigin, + betaOrigin, + ungroupedOrigin, + ]); + expect(result.get(betaOrigin)).toBe(result.get(alphaOrigin)); + expect(result.get(ungroupedOrigin)).not.toBe(result.get(alphaOrigin)); + expect(result.get(alphaOrigin)).toBe( + await authService.appUidFromOrigin(alphaOrigin), + ); + }); }); describe('AuthService.subdomainOwnerIdFromOrigin — edge cases', () => { diff --git a/src/backend/services/auth/AuthService.ts b/src/backend/services/auth/AuthService.ts index dfc313215..6791c3eb6 100644 --- a/src/backend/services/auth/AuthService.ts +++ b/src/backend/services/auth/AuthService.ts @@ -819,6 +819,102 @@ export class AuthService extends PuterService { return this.#derivedAppUidForOrigin(appOrigin); } + /** + * Batched sibling of {@link appUidFromOrigin} for listing paths that need a + * uid per origin without a round trip each. Unparseable or blocked origins + * resolve to null instead of throwing. + */ + async appUidsFromOrigins( + origins: string[], + ): Promise> { + const result = new Map(); + const uniqueOrigins = [...new Set(origins)]; + + const appOriginByOrigin = new Map(); + await Promise.all( + uniqueOrigins.map(async (origin) => { + try { + appOriginByOrigin.set( + origin, + await this.#appOriginFor(origin), + ); + } catch { + appOriginByOrigin.set(origin, null); + } + }), + ); + + const uniqueAppOrigins = [ + ...new Set( + [...appOriginByOrigin.values()].filter( + (o): o is string => o !== null, + ), + ), + ]; + const blockedByAppOrigin = new Map(); + await Promise.all( + uniqueAppOrigins.map(async (appOrigin) => { + try { + const block = + await this.services.appOriginBlocklist.isOriginBlocked( + appOrigin, + ); + blockedByAppOrigin.set(appOrigin, block.blocked); + } catch { + blockedByAppOrigin.set(appOrigin, true); + } + }), + ); + + const resolvableAppOrigins = uniqueAppOrigins.filter( + (appOrigin) => !blockedByAppOrigin.get(appOrigin), + ); + const canonicalByAppOrigin = + await this.#findCanonicalAppUidsForOrigins(resolvableAppOrigins); + + // Gen-0 derived uid per app origin that missed the canonical lookup, + // prefetched in one batch so `#derivedAppUidForOrigin`'s loop doesn't + // pay a round trip for the common (no repoint) case. + const missedAppOrigins = resolvableAppOrigins.filter( + (appOrigin) => !canonicalByAppOrigin.get(appOrigin), + ); + const gen0ByAppOrigin = new Map(); + for (const appOrigin of missedAppOrigins) { + gen0ByAppOrigin.set(appOrigin, this.#originUid(appOrigin, 0)); + } + const gen0Uids = [...new Set(gen0ByAppOrigin.values())]; + const prefetched = + gen0Uids.length > 0 + ? await this.stores.app.getByUids(gen0Uids) + : new Map(); + + const derivedByAppOrigin = new Map(); + for (const appOrigin of missedAppOrigins) { + const gen0Uid = gen0ByAppOrigin.get(appOrigin)!; + const uid = await this.#derivedAppUidForOrigin(appOrigin, (uid) => + uid === gen0Uid + ? Promise.resolve(prefetched.get(uid) ?? null) + : this.stores.app.getByUid(uid), + ); + derivedByAppOrigin.set(appOrigin, uid); + } + + for (const origin of origins) { + const appOrigin = appOriginByOrigin.get(origin) ?? null; + if (appOrigin === null || blockedByAppOrigin.get(appOrigin)) { + result.set(origin, null); + continue; + } + const canonicalUid = canonicalByAppOrigin.get(appOrigin); + result.set( + origin, + canonicalUid ?? derivedByAppOrigin.get(appOrigin) ?? null, + ); + } + + return result; + } + /** * Normalized origin of `url` with aliased hosts and hosting-domain variants * collapsed, after `app.from-origin` listeners have rewritten it. Null when @@ -834,17 +930,29 @@ export class AuthService extends PuterService { return event.origin; } + /** `app-` for generation `gen` (0 is the first-visit uid). */ + #originUid(origin: string, gen: number): string { + const name = gen === 0 ? origin : `${origin}#${gen}`; + return `app-${uuidv5(name, APP_ORIGIN_UUID_NAMESPACE)}`; + } + /** * `app-`, unless that uid's row now lives at another * origin. A repointed row keeps its uid (and the data and grants keyed to * it), so the origin it left moves on to a successor uid instead of - * resolving to someone's app it no longer serves. + * resolving to someone's app it no longer serves. `getByUid` is injectable + * so a batch caller can serve prefetched rows. */ - async #derivedAppUidForOrigin(origin: string): Promise { + async #derivedAppUidForOrigin( + origin: string, + getByUid: ( + uid: string, + ) => Promise<{ index_url?: unknown } | null | undefined> = (uid) => + this.stores.app.getByUid(uid), + ): Promise { for (let gen = 0; gen <= MAX_ORIGIN_UID_GENERATIONS; gen++) { - const name = gen === 0 ? origin : `${origin}#${gen}`; - const uid = `app-${uuidv5(name, APP_ORIGIN_UUID_NAMESPACE)}`; - const app = await this.stores.app.getByUid(uid); + const uid = this.#originUid(origin, gen); + const app = await getByUid(uid); if (!app) return uid; if ( typeof app.index_url === 'string' && @@ -1058,21 +1166,17 @@ export class AuthService extends PuterService { } /** - * Find the real app row whose `index_url` canonically matches `origin`. - * - * Build candidate URLs from the origin's subdomain crossed with every - * configured hosting domain (static + private, with and without ports). - * Prefer private rows, then the oldest match, for deterministic - * tie-breaking across historically-duplicated rows. + * Every `index_url` string that would canonically match `origin` — the + * origin's subdomain crossed with every configured hosting domain (static + * and private, with and without ports), alias-group hosts, and protocol + * variants. */ - async #findCanonicalAppUidForOrigin( - origin: string, - ): Promise { + #indexUrlCandidatesForOrigin(origin: string): string[] { let parsed: URL; try { parsed = new URL(origin); } catch { - return null; + return []; } const config = this.config as { protocol?: string }; @@ -1120,18 +1224,103 @@ export class AuthService extends PuterService { urlCandidates.push(base, `${base}/`, `${base}/index.html`); } } - const uniqueCandidates = [...new Set(urlCandidates)]; - if (uniqueCandidates.length === 0) return null; + return [...new Set(urlCandidates)]; + } - const placeholders = uniqueCandidates.map(() => '?').join(', '); - // Private rows win over public duplicates; oldest id breaks ties. - const rows = (await this.clients.db.read( - `SELECT \`uid\` FROM \`apps\` WHERE \`index_url\` IN (${placeholders}) ` + - `ORDER BY CASE WHEN \`is_private\` = ${this.clients.db.booleanLiteral(true)} THEN 0 ELSE 1 END, \`id\` ASC LIMIT 1`, - uniqueCandidates, - )) as Array<{ uid?: string }>; - const uid = rows[0]?.uid; - return typeof uid === 'string' && uid ? uid : null; + /** + * Uid of the real app row whose `index_url` canonically matches `origin`; + * private rows first, then the oldest, across historical duplicates. + */ + async #findCanonicalAppUidForOrigin( + origin: string, + ): Promise { + return this.stores.app.findCanonicalUidByIndexUrlCandidates( + this.#indexUrlCandidatesForOrigin(origin), + ); + } + + /** + * Batched {@link #findCanonicalAppUidForOrigin}, keyed by the origins passed + * in, with the same private-first, oldest-id rule. + */ + async #findCanonicalAppUidsForOrigins( + origins: string[], + ): Promise> { + const result = new Map(); + const candidateSetByOrigin = new Map>(); + const allCandidates = new Set(); + + for (const origin of origins) { + const candidates = this.#indexUrlCandidatesForOrigin(origin); + candidateSetByOrigin.set( + origin, + new Set(candidates.map((c) => c.toLowerCase())), + ); + for (const c of candidates) allCandidates.add(c); + } + + if (allCandidates.size === 0) { + for (const origin of origins) result.set(origin, null); + return result; + } + + const winners = (await this.stores.app.listIndexUrlWinners([ + ...allCandidates, + ])) as Array<{ + index_url: unknown; + private_id: unknown; + min_id: unknown; + }>; + const winnerByIndexUrl = new Map(); + for (const winner of winners) { + if (typeof winner.index_url === 'string') { + winnerByIndexUrl.set(winner.index_url.toLowerCase(), winner); + } + } + + // Lowest private id across the origin's matching groups, else lowest id. + const winnerIdByOrigin = new Map(); + for (const origin of origins) { + const candidateSet = candidateSetByOrigin.get(origin); + if (!candidateSet) continue; + + let bestPrivateId: number | null = null; + let bestMinId: number | null = null; + for (const candidate of candidateSet) { + const winner = winnerByIndexUrl.get(candidate); + if (!winner) continue; + if (winner.private_id != null) { + const id = Number(winner.private_id); + if (bestPrivateId === null || id < bestPrivateId) { + bestPrivateId = id; + } + } + if (winner.min_id != null) { + const id = Number(winner.min_id); + if (bestMinId === null || id < bestMinId) { + bestMinId = id; + } + } + } + const winnerId = bestPrivateId ?? bestMinId; + if (winnerId !== null) winnerIdByOrigin.set(origin, winnerId); + } + + const uniqueIds = [...new Set(winnerIdByOrigin.values())]; + // Aggregates can come back as strings; they were Number()-ed above to + // match `getByIds` keys. + const appsById = + uniqueIds.length > 0 + ? await this.stores.app.getByIds(uniqueIds) + : new Map(); + + for (const origin of origins) { + const id = winnerIdByOrigin.get(origin); + const uid = id !== undefined ? appsById.get(id)?.uid : undefined; + result.set(origin, typeof uid === 'string' && uid ? uid : null); + } + + return result; } async getUserAppToken(actor: Actor, appUid: string): Promise { diff --git a/src/backend/stores/app/AppStore.js b/src/backend/stores/app/AppStore.js index 4c79f973c..143b935e3 100644 --- a/src/backend/stores/app/AppStore.js +++ b/src/backend/stores/app/AppStore.js @@ -58,6 +58,8 @@ const OLD_APP_NAME_TTL_MONTHS = 3; // limit is 999; staying well under that keeps `getByIds` portable across // backends without splitting the cap by driver. const BULK_QUERY_CHUNK_SIZE = 200; +// Placeholders per index_url `IN (…)` chunk; under SQLite's old 999 default. +const INDEX_URL_CHUNK_SIZE = 900; // Top-level all-time open/user counts: hot path, slow to compute. Cached // lazily on read (pipelined MGET on every app list/read; misses query the @@ -299,15 +301,105 @@ export class AppStore extends PuterStore { /** * Every app whose `index_url` matches one of `candidates`, with the owner * and the app that built it. Used by the subdomain driver to decide who may - * re-create a hosted name that apps still point at. + * re-create a hosted name that apps still point at, and to derive a + * subdomain row's associated app. */ async listByIndexUrlCandidates(candidates) { - if (!Array.isArray(candidates) || candidates.length === 0) return []; - const placeholders = candidates.map(() => '?').join(', '); - return this.clients.db.read( - `SELECT \`id\`, \`uid\`, \`owner_user_id\`, \`app_owner\` FROM \`apps\` WHERE \`index_url\` IN (${placeholders})`, - [...candidates], + const uniqueCandidates = [ + ...new Set( + (Array.isArray(candidates) ? candidates : []).filter( + (c) => typeof c === 'string' && c.length > 0, + ), + ), + ]; + if (uniqueCandidates.length === 0) return []; + + const rowsById = new Map(); + for ( + let offset = 0; + offset < uniqueCandidates.length; + offset += INDEX_URL_CHUNK_SIZE + ) { + const chunk = uniqueCandidates.slice( + offset, + offset + INDEX_URL_CHUNK_SIZE, + ); + const placeholders = chunk.map(() => '?').join(', '); + const rows = await this.clients.db.read( + `SELECT \`id\`, \`uid\`, \`owner_user_id\`, \`app_owner\`, \`index_url\` FROM \`apps\` WHERE \`index_url\` IN (${placeholders})`, + chunk, + ); + for (const row of rows) rowsById.set(row.id, row); + } + return [...rowsById.values()]; + } + + /** + * Canonical app uid among rows matching `candidates`: private first, then + * oldest. + */ + async findCanonicalUidByIndexUrlCandidates(candidates) { + const uniqueCandidates = [ + ...new Set( + (Array.isArray(candidates) ? candidates : []).filter( + (c) => typeof c === 'string' && c.length > 0, + ), + ), + ]; + if (uniqueCandidates.length === 0) return null; + + const placeholders = uniqueCandidates.map(() => '?').join(', '); + const rows = await this.clients.db.read( + `SELECT \`uid\` FROM \`apps\` WHERE \`index_url\` IN (${placeholders}) ` + + `ORDER BY CASE WHEN \`is_private\` = ${this.clients.db.booleanLiteral(true)} THEN 0 ELSE 1 END, \`id\` ASC LIMIT 1`, + uniqueCandidates, ); + const uid = rows[0]?.uid; + return typeof uid === 'string' && uid ? uid : null; + } + + /** + * One row per distinct `index_url` in `candidates` with its lowest private + * id (`private_id`, null if none) and lowest id (`min_id`). An external + * index_url can be shared by thousands of apps, so batch callers pick + * canonical winners from these instead of every matching row. + */ + async listIndexUrlWinners(candidates) { + const uniqueCandidates = [ + ...new Set( + (Array.isArray(candidates) ? candidates : []).filter( + (c) => typeof c === 'string' && c.length > 0, + ), + ), + ]; + if (uniqueCandidates.length === 0) return []; + + const winnersByIndexUrl = new Map(); + for ( + let offset = 0; + offset < uniqueCandidates.length; + offset += INDEX_URL_CHUNK_SIZE + ) { + const chunk = uniqueCandidates.slice( + offset, + offset + INDEX_URL_CHUNK_SIZE, + ); + const placeholders = chunk.map(() => '?').join(', '); + const rows = await this.clients.db.read( + `SELECT \`index_url\`, ` + + `MIN(CASE WHEN \`is_private\` = ${this.clients.db.booleanLiteral(true)} THEN \`id\` END) AS private_id, ` + + `MIN(\`id\`) AS min_id ` + + `FROM \`apps\` WHERE \`index_url\` IN (${placeholders}) GROUP BY \`index_url\``, + chunk, + ); + // Deduped candidates land in one chunk each, so groups never span chunks. + for (const row of rows) { + if (typeof row.index_url === 'string') { + winnersByIndexUrl.set(row.index_url, row); + } + } + } + return [...winnersByIndexUrl.values()]; } /** diff --git a/src/backend/stores/app/AppStore.test.js b/src/backend/stores/app/AppStore.test.js index b46e82ebb..d4424eb7b 100644 --- a/src/backend/stores/app/AppStore.test.js +++ b/src/backend/stores/app/AppStore.test.js @@ -1419,3 +1419,168 @@ describe('AppStore deleted-row tombstones', () => { expect(await redis.get(`apps:uid:${uid}`)).not.toBeNull(); }); }); + +describe('AppStore listByIndexUrlCandidates', () => { + let server; + let appStore; + + beforeAll(async () => { + server = await setupTestServer(); + appStore = server.stores.app; + // Shared mock redis persists `apps:id:*` across servers while each + // fresh sqlite db restarts its id sequence at 1 — without this a + // freshly-created row can read back a stale cached row at the same id. + await clearAppCache(server.clients.redis); + }); + + afterAll(async () => { + await server?.shutdown(); + }); + + const createApp = async (indexUrl) => { + const name = `idxurl-${Math.random().toString(36).slice(2, 10)}`; + return appStore.create( + { name, title: name, index_url: indexUrl }, + { ownerUserId: 1 }, + ); + }; + + it('chunks a candidate list spanning more than one page and still finds the real URL at the end', async () => { + const real = await createApp('https://real-target.example.com/'); + // 2,000 candidates is more than one 900-wide chunk; the real URL + // sits past the first chunk boundary. + const candidates = []; + for (let i = 0; i < 2000; i++) { + candidates.push(`https://decoy-${i}.example.com/`); + } + candidates.push(real.index_url); + + const rows = await appStore.listByIndexUrlCandidates(candidates); + expect(rows.map((r) => r.id)).toEqual([real.id]); + }); + + it('dedupes a candidate repeated across chunk boundaries to one row', async () => { + const real = await createApp('https://dup-target.example.com/'); + const candidates = Array.from({ length: 2000 }, () => real.index_url); + + const rows = await appStore.listByIndexUrlCandidates(candidates); + expect(rows.length).toBe(1); + expect(rows[0].id).toBe(real.id); + }); + + it('returns [] for an empty or non-array candidate list', async () => { + expect(await appStore.listByIndexUrlCandidates([])).toEqual([]); + expect(await appStore.listByIndexUrlCandidates(null)).toEqual([]); + }); +}); + +describe('AppStore findCanonicalUidByIndexUrlCandidates', () => { + let server; + let appStore; + + beforeAll(async () => { + server = await setupTestServer(); + appStore = server.stores.app; + await clearAppCache(server.clients.redis); + }); + + afterAll(async () => { + await server?.shutdown(); + }); + + it('prefers the private row, then the oldest match', async () => { + const url = `https://canon-${Math.random().toString(36).slice(2, 10)}.example.com/`; + const pub = await appStore.create( + { name: `pub-${Math.random().toString(36).slice(2, 8)}`, title: 't', index_url: url }, + { ownerUserId: 1 }, + ); + const privUid = `app-${Math.random().toString(36).slice(2, 10)}`; + await server.clients.db.write( + 'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `is_private`) VALUES (?, ?, ?, ?, ?)', + [privUid, `priv-${Math.random().toString(36).slice(2, 8)}`, 't', url, 1], + ); + + const uid = await appStore.findCanonicalUidByIndexUrlCandidates([url]); + expect(uid).toBe(privUid); + expect(uid).not.toBe(pub.uid); + }); + + it('returns null for an empty, non-array, or non-matching candidate list', async () => { + expect(await appStore.findCanonicalUidByIndexUrlCandidates([])).toBeNull(); + expect(await appStore.findCanonicalUidByIndexUrlCandidates(null)).toBeNull(); + expect( + await appStore.findCanonicalUidByIndexUrlCandidates([ + 'https://nothing-here.example.com/', + ]), + ).toBeNull(); + }); +}); + +describe('AppStore listIndexUrlWinners', () => { + let server; + let appStore; + + beforeAll(async () => { + server = await setupTestServer(); + appStore = server.stores.app; + await clearAppCache(server.clients.redis); + }); + + afterAll(async () => { + await server?.shutdown(); + }); + + const createApp = async (indexUrl) => { + const name = `winner-${Math.random().toString(36).slice(2, 10)}`; + return appStore.create( + { name, title: name, index_url: indexUrl }, + { ownerUserId: 1 }, + ); + }; + + it('collapses many apps sharing one index_url into a single winner row', async () => { + // An external dev-server default — the exact shape that can be + // shared by thousands of apps and would blow up a row-per-app query. + const sharedUrl = `http://localhost:${5000 + Math.floor(Math.random() * 1000)}/`; + for (let i = 0; i < 50; i++) { + await createApp(sharedUrl); + } + const otherUrl = `https://distinct-${Math.random().toString(36).slice(2, 10)}.example.com/`; + await createApp(otherUrl); + + const rows = await appStore.listIndexUrlWinners([sharedUrl, otherUrl]); + expect(rows.length).toBe(2); + // One row per distinct index_url, never one per matching app. + expect(rows.length).toBeLessThanOrEqual(2); + const shared = rows.find((r) => r.index_url === sharedUrl); + expect(shared).toBeTruthy(); + expect(shared.private_id == null).toBe(true); + expect(Number(shared.min_id)).toBeGreaterThan(0); + }); + + it('reports the lowest private id and the lowest id overall per index_url', async () => { + const url = `https://winner-${Math.random().toString(36).slice(2, 10)}.example.com/`; + const pub = await createApp(url); + const priv1Uid = `app-${Math.random().toString(36).slice(2, 10)}`; + await server.clients.db.write( + 'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `is_private`) VALUES (?, ?, ?, ?, ?)', + [priv1Uid, `priv1-${Math.random().toString(36).slice(2, 8)}`, 't', url, 1], + ); + const priv1 = await appStore.getByUid(priv1Uid); + const priv2Uid = `app-${Math.random().toString(36).slice(2, 10)}`; + await server.clients.db.write( + 'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `is_private`) VALUES (?, ?, ?, ?, ?)', + [priv2Uid, `priv2-${Math.random().toString(36).slice(2, 8)}`, 't', url, 1], + ); + const priv2 = await appStore.getByUid(priv2Uid); + + const [winner] = await appStore.listIndexUrlWinners([url]); + expect(Number(winner.min_id)).toBe(pub.id); + expect(Number(winner.private_id)).toBe(Math.min(priv1.id, priv2.id)); + }); + + it('returns [] for an empty or non-array candidate list', async () => { + expect(await appStore.listIndexUrlWinners([])).toEqual([]); + expect(await appStore.listIndexUrlWinners(null)).toEqual([]); + }); +}); diff --git a/src/backend/stores/subdomain/SubdomainStore.test.ts b/src/backend/stores/subdomain/SubdomainStore.test.ts index 8456922aa..a0fb87d78 100644 --- a/src/backend/stores/subdomain/SubdomainStore.test.ts +++ b/src/backend/stores/subdomain/SubdomainStore.test.ts @@ -563,3 +563,90 @@ describe('SubdomainStore reads and writes', () => { ).resolves.toBeUndefined(); }); }); + +// -- getBySubdomains (batched) -- + +describe('SubdomainStore.getBySubdomains', () => { + it('is keyed by the requested names and omits names with no row', async () => { + const owner = await makeUser(); + const a = `gbs-a-${Math.random().toString(36).slice(2, 8)}`; + const b = `gbs-b-${Math.random().toString(36).slice(2, 8)}`; + const missing = `gbs-missing-${Math.random().toString(36).slice(2, 8)}`; + await store.create({ userId: owner, subdomain: a } as never); + await store.create({ userId: owner, subdomain: b } as never); + + const found = await store.getBySubdomains([a, b, missing]); + + expect(found.size).toBe(2); + expect(found.get(a)?.subdomain).toBe(a); + expect(found.get(b)?.subdomain).toBe(b); + expect(found.has(missing)).toBe(false); + }); + + it('serves a cache hit with no DB read', async () => { + const owner = await makeUser(); + const name = `gbs-cached-${Math.random().toString(36).slice(2, 8)}`; + // `create` writes through the cache, so the row is already warm. + await store.create({ userId: owner, subdomain: name } as never); + + const read = vi.spyOn(server.clients.db, 'read'); + const pread = vi.spyOn(server.clients.db, 'pread'); + try { + const found = await store.getBySubdomains([name]); + expect(found.get(name)?.subdomain).toBe(name); + expect(read).not.toHaveBeenCalled(); + expect(pread).not.toHaveBeenCalled(); + } finally { + read.mockRestore(); + pread.mockRestore(); + } + }); + + it('a cached negative marker is absent from the result with no DB read', async () => { + const name = `gbs-neg-${Math.random().toString(36).slice(2, 8)}`; + // Poison the negative-cache marker first. + expect(await store.getBySubdomains([name])).toEqual(new Map()); + + const read = vi.spyOn(server.clients.db, 'read'); + try { + const found = await store.getBySubdomains([name]); + expect(found.has(name)).toBe(false); + expect(read).not.toHaveBeenCalled(); + } finally { + read.mockRestore(); + } + }); + + it('treats a cached JSON null as a miss and reads the DB, like getBySubdomain', async () => { + const owner = await makeUser(); + const name = `gbs-null-${Math.random().toString(36).slice(2, 8)}`; + await store.create({ userId: owner, subdomain: name } as never); + await server.clients.redis.set(cacheKey(name), 'null'); + + const found = await store.getBySubdomains([name]); + expect(found.get(name)?.subdomain).toBe(name); + }); + + it('a primary read uses pread and writes through, healing a stale negative marker', async () => { + const owner = await makeUser(); + const name = `gbs-heal-${Math.random().toString(36).slice(2, 8)}`; + // Poison the cache with a negative marker, then create the row. + expect(await store.getBySubdomains([name])).toEqual(new Map()); + await store.create({ userId: owner, subdomain: name } as never); + + const pread = vi.spyOn(server.clients.db, 'pread'); + try { + const primaryFound = await store.getBySubdomains([name], { + primary: true, + }); + expect(primaryFound.get(name)?.subdomain).toBe(name); + expect(pread).toHaveBeenCalled(); + } finally { + pread.mockRestore(); + } + + // The write-through heals the replica-path cache too. + const healedFound = await store.getBySubdomains([name]); + expect(healedFound.get(name)?.subdomain).toBe(name); + }); +}); diff --git a/src/backend/stores/subdomain/SubdomainStore.ts b/src/backend/stores/subdomain/SubdomainStore.ts index c60a9e65c..12983c990 100644 --- a/src/backend/stores/subdomain/SubdomainStore.ts +++ b/src/backend/stores/subdomain/SubdomainStore.ts @@ -111,6 +111,8 @@ const CACHE_TTL_SECONDS = 60 * 60; // Sentinel so 404s on the same public subdomain don't hit the DB repeatedly. const NEGATIVE_CACHE_MARKER = '__none__'; const NEGATIVE_CACHE_TTL_SECONDS = 10; +// Cap on placeholders per `IN (?, ?, …)` chunk — mirrors AppStore/UserStore. +const BULK_QUERY_CHUNK_SIZE = 200; export class SubdomainStore extends PuterStore { // -- Reads -------------------------------------------------------- @@ -170,34 +172,115 @@ export class SubdomainStore extends PuterStore { : await this.clients.db.read(sql, [subdomain]); const row = (rows[0] as unknown as SubdomainRow | undefined) ?? null; - // These writes are fire-and-forget, so a mutation that lands between - // the SELECT above and the SET below would otherwise be overwritten - // by the row we just read — stranding the pre-write row in cache for - // the full TTL (an hour of a site serving its old root_dir after - // `hosting.update`). A replica read is not authoritative, so it only - // *populates* an absent key (`NX`) and can never clobber a fresher - // write. A `primary` read is read-after-write on the primary, so it - // writes through — that's what heals a stale negative marker. - const populate = (value: string, ttlSeconds: number) => - (primary - ? this.clients.redis.set(cacheKey, value, 'EX', ttlSeconds) - : this.clients.redis.set( - cacheKey, - value, - 'EX', - ttlSeconds, - 'NX', - ) - ).catch(() => {}); - if (row) { - populate(JSON.stringify(row), CACHE_TTL_SECONDS); + void this.#populateCache( + cacheKey, + JSON.stringify(row), + CACHE_TTL_SECONDS, + primary, + ); } else { - populate(NEGATIVE_CACHE_MARKER, NEGATIVE_CACHE_TTL_SECONDS); + void this.#populateCache( + cacheKey, + NEGATIVE_CACHE_MARKER, + NEGATIVE_CACHE_TTL_SECONDS, + primary, + ); } return row; } + /** + * Batched sibling of {@link getBySubdomain} — one cache pipeline plus one + * chunked `IN (…)` query for the misses, instead of one lookup per name. + * Keyed by the requested names; a name with no row (negative-cached or + * genuinely absent) is simply missing from the returned map. + */ + async getBySubdomains( + names: string[], + { primary = false }: { primary?: boolean } = {}, + ): Promise> { + const result = new Map(); + const uniqueNames = [ + ...new Set( + (Array.isArray(names) ? names : []).filter( + (n): n is string => typeof n === 'string' && n.length > 0, + ), + ), + ]; + if (uniqueNames.length === 0) return result; + + let missingNames = uniqueNames; + if (!primary) { + missingNames = []; + try { + const pipeline = this.clients.redis.pipeline(); + for (const name of uniqueNames) { + pipeline.get(this.#cacheKey(name)); + } + const cacheResults = (await pipeline.exec()) ?? []; + for (let i = 0; i < uniqueNames.length; i++) { + const name = uniqueNames[i]!; + const raw = cacheResults[i]?.[1]; + if (raw === NEGATIVE_CACHE_MARKER) continue; // cached miss + if (typeof raw === 'string') { + try { + const parsed = JSON.parse( + raw, + ) as SubdomainRow | null; + if (parsed) { + result.set(name, parsed); + continue; + } + } catch { + // Fall through to DB on any parse failure. + } + } + missingNames.push(name); + } + } catch { + missingNames = uniqueNames; + } + } + if (missingNames.length === 0) return result; + + const rowsByName = new Map(); + for ( + let offset = 0; + offset < missingNames.length; + offset += BULK_QUERY_CHUNK_SIZE + ) { + const chunk = missingNames.slice( + offset, + offset + BULK_QUERY_CHUNK_SIZE, + ); + const placeholders = chunk.map(() => '?').join(', '); + const sql = `SELECT * FROM \`subdomains\` WHERE \`subdomain\` IN (${placeholders})`; + const rows = (primary + ? await this.clients.db.pread(sql, chunk) + : await this.clients.db.read( + sql, + chunk, + )) as unknown as SubdomainRow[]; + // Keyed lowercase: a case-insensitive collation can return a row + // cased differently from the name, which `getBySubdomain` accepts. + for (const row of rows) { + if (typeof row.subdomain !== 'string') continue; + const key = row.subdomain.toLowerCase(); + if (!rowsByName.has(key)) rowsByName.set(key, row); + } + } + + for (const name of missingNames) { + const row = rowsByName.get(name.toLowerCase()); + if (row) result.set(name, row); + } + + void this.#populateCacheForNames(missingNames, result, primary); + + return result; + } + async listByUserId( userId: number, { @@ -634,6 +717,61 @@ export class SubdomainStore extends PuterStore { return `${CACHE_KEY_PREFIX}:listByUserPrefixKeys:${userId}`; } + /** + * Write-through on a primary read (heals a stale negative marker); `NX` on + * a replica read so an in-flight fresher write can't be clobbered by a + * stale value landing after it. + */ + async #populateCache( + cacheKey: string, + value: string, + ttlSeconds: number, + primary: boolean, + ): Promise { + try { + if (primary) { + await this.clients.redis.set(cacheKey, value, 'EX', ttlSeconds); + } else { + await this.clients.redis.set( + cacheKey, + value, + 'EX', + ttlSeconds, + 'NX', + ); + } + } catch { + /* best-effort */ + } + } + + /** Pipelined sibling of {@link #populateCache} for `getBySubdomains`. */ + async #populateCacheForNames( + names: string[], + rowsByName: Map, + primary: boolean, + ): Promise { + try { + const pipeline = this.clients.redis.pipeline(); + for (const name of names) { + const row = rowsByName.get(name); + const cacheKey = this.#cacheKey(name); + const value = row ? JSON.stringify(row) : NEGATIVE_CACHE_MARKER; + const ttl = row + ? CACHE_TTL_SECONDS + : NEGATIVE_CACHE_TTL_SECONDS; + if (primary) { + pipeline.set(cacheKey, value, 'EX', ttl); + } else { + pipeline.set(cacheKey, value, 'EX', ttl, 'NX'); + } + } + await pipeline.exec(); + } catch { + /* best-effort */ + } + } + async #refreshCache(row: { subdomain?: string }) { if (!row?.subdomain) return; await this.publishCacheKeys({ diff --git a/src/backend/util/hostedAppBacking.test.ts b/src/backend/util/hostedAppBacking.test.ts index 8ed6745f7..fc8bb1258 100644 --- a/src/backend/util/hostedAppBacking.test.ts +++ b/src/backend/util/hostedAppBacking.test.ts @@ -17,13 +17,17 @@ * along with this program. If not, see . */ -import { describe, expect, it, vi } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { v4 as uuidv4 } from 'uuid'; +import { PuterServer } from '../server.js'; +import { setupTestServer } from '../testUtil.js'; import { buildHostedBackingDenial, buildHostedSubdomainIndexUrlCandidates, extractPuterHostedSubdomain, getPuterHostedDomains, hostedIndexUrlBackingIsUnavailable, + hostedIndexUrlBackingsAreUnavailable, } from './hostedAppBacking.js'; // Pure-unit companion to the integration coverage in @@ -354,3 +358,159 @@ describe('buildHostedBackingDenial', () => { }); }); }); + +// Matches the hosting domains in config.default.json, so subdomain rows +// created through the real store resolve the same way `AppDriver` would see +// them in production. +const HOSTING_CONFIG = { + static_hosting_domain: 'site.puter.localhost', + static_hosting_domain_alt: 'host.puter.localhost', + private_app_hosting_domain: 'app.puter.localhost', + private_app_hosting_domain_alt: 'dev.puter.localhost', +}; +const hostedUrl = (sub: string) => `https://${sub}.site.puter.localhost/`; + +describe('hostedIndexUrlBackingsAreUnavailable (batched, against a real store)', () => { + let server: PuterServer; + + beforeAll(async () => { + server = await setupTestServer(); + }); + + afterAll(async () => { + await server?.shutdown(); + }); + + const makeUser = async () => { + const username = `hab-${Math.random().toString(36).slice(2, 10)}`; + return server.stores.user.create({ + username, + uuid: uuidv4(), + password: null, + email: `${username}@test.local`, + free_storage: 100 * 1024 * 1024, + requires_email_confirmation: false, + }); + }; + + it('resolves a mix of apps in input order with one batched call', async () => { + const owner = await makeUser(); + const attacker = await makeUser(); + + const liveSub = `live-${Math.random().toString(36).slice(2, 8)}`; + await server.stores.subdomain.create({ + userId: owner.id, + subdomain: liveSub, + }); + const reclaimedSub = `reclaimed-${Math.random().toString(36).slice(2, 8)}`; + await server.stores.subdomain.create({ + userId: attacker.id, + subdomain: reclaimedSub, + }); + const danglingSub = `dangling-${Math.random().toString(36).slice(2, 8)}`; + + const external = { + index_url: 'https://elsewhere.example.com/', + owner_user_id: owner.id, + }; + const live = { index_url: hostedUrl(liveSub), owner_user_id: owner.id }; + const reclaimed = { + index_url: hostedUrl(reclaimedSub), + owner_user_id: owner.id, + }; + const dangling = { + index_url: hostedUrl(danglingSub), + owner_user_id: owner.id, + }; + + const getBySubdomains = vi.spyOn( + server.stores.subdomain, + 'getBySubdomains', + ); + try { + const results = await hostedIndexUrlBackingsAreUnavailable({ + apps: [external, live, reclaimed, dangling], + subdomainStore: server.stores.subdomain, + config: HOSTING_CONFIG, + }); + // Input order preserved; external → false, live → false, a + // reclaimed owner → true, a gone subdomain → true. + expect(results).toEqual([false, false, true, true]); + + // One non-primary batch carrying only the three hosted names — + // the external app never touches the store. + expect(getBySubdomains).toHaveBeenCalledTimes(2); + expect([...getBySubdomains.mock.calls[0]![0]].sort()).toEqual( + [liveSub, reclaimedSub, danglingSub].sort(), + ); + // Primary follow-up only for the one name still missing. + expect(getBySubdomains.mock.calls[1]).toEqual([ + [danglingSub], + { primary: true }, + ]); + } finally { + getBySubdomains.mockRestore(); + } + }); + + it('makes no store call when every app is non-hosted', async () => { + const owner = await makeUser(); + const external = { + index_url: 'https://elsewhere.example.com/', + owner_user_id: owner.id, + }; + const builtin = { index_url: undefined, owner_user_id: owner.id }; + + const getBySubdomains = vi.spyOn( + server.stores.subdomain, + 'getBySubdomains', + ); + try { + const results = await hostedIndexUrlBackingsAreUnavailable({ + apps: [external, builtin], + subdomainStore: server.stores.subdomain, + config: HOSTING_CONFIG, + }); + expect(results).toEqual([false, false]); + expect(getBySubdomains).not.toHaveBeenCalled(); + } finally { + getBySubdomains.mockRestore(); + } + }); + + it('does exactly one primary batch covering every still-missing name', async () => { + const owner = await makeUser(); + const goneA = `gone-a-${Math.random().toString(36).slice(2, 6)}`; + const goneB = `gone-b-${Math.random().toString(36).slice(2, 6)}`; + const danglingA = { + index_url: hostedUrl(goneA), + owner_user_id: owner.id, + }; + const danglingB = { + index_url: hostedUrl(goneB), + owner_user_id: owner.id, + }; + + const getBySubdomains = vi.spyOn( + server.stores.subdomain, + 'getBySubdomains', + ); + try { + const results = await hostedIndexUrlBackingsAreUnavailable({ + apps: [danglingA, danglingB], + subdomainStore: server.stores.subdomain, + config: HOSTING_CONFIG, + }); + expect(results).toEqual([true, true]); + expect(getBySubdomains).toHaveBeenCalledTimes(2); + expect(getBySubdomains.mock.calls[1]![1]).toEqual({ + primary: true, + }); + expect([...getBySubdomains.mock.calls[1]![0]].sort()).toEqual( + [goneA, goneB].sort(), + ); + } finally { + getBySubdomains.mockRestore(); + } + }); +}); diff --git a/src/backend/util/hostedAppBacking.ts b/src/backend/util/hostedAppBacking.ts index e9010db10..d7b966c25 100644 --- a/src/backend/util/hostedAppBacking.ts +++ b/src/backend/util/hostedAppBacking.ts @@ -181,6 +181,24 @@ export function extractPuterHostedSubdomain( return null; } +/** + * True when `row` (the subdomain backing `app`'s hosted index_url, or null) + * means the app's launch is unsafe: gone, or reclaimed by a different owner. + */ +function backingRowIsUnavailable( + app: AppBackingRow, + row: { user_id?: unknown } | null, +): boolean { + if (!row) return true; // subdomain no longer exists → dangling + + const appOwnerId = Number(app.owner_user_id); + const subdomainOwnerId = Number(row.user_id); + if (!Number.isInteger(appOwnerId) || !Number.isInteger(subdomainOwnerId)) { + return true; + } + return subdomainOwnerId !== appOwnerId; +} + /** * True when the app's puter-hosted subdomain is missing, or is currently owned * by a different user than the app's owner (it was reclaimed — launching would @@ -208,14 +226,46 @@ export async function hostedIndexUrlBackingIsUnavailable({ primary: true, }); } - if (!row) return true; // subdomain no longer exists → dangling + return backingRowIsUnavailable(app, row); +} - const appOwnerId = Number(app.owner_user_id); - const subdomainOwnerId = Number(row.user_id); - if (!Number.isInteger(appOwnerId) || !Number.isInteger(subdomainOwnerId)) { - return true; +/** + * Batched sibling of {@link hostedIndexUrlBackingIsUnavailable} — one + * `getBySubdomains` call (plus a single primary follow-up for whatever's still + * missing) instead of per-app round trips. Returns a boolean per `apps` entry, + * in input order, with the same semantics as the single-app function above. + */ +export async function hostedIndexUrlBackingsAreUnavailable({ + apps, + subdomainStore, + config, +}: { + apps: AppBackingRow[]; + subdomainStore: Pick; + config: HostedDomainConfig | undefined | null; +}): Promise { + const subdomains = apps.map((app) => + extractPuterHostedSubdomain(app.index_url, config), + ); + const names = [ + ...new Set(subdomains.filter((s): s is string => s !== null)), + ]; + if (names.length === 0) return apps.map(() => false); + + let rowsByName = await subdomainStore.getBySubdomains(names); + const missing = names.filter((name) => !rowsByName.has(name)); + if (missing.length > 0) { + const primaryRows = await subdomainStore.getBySubdomains(missing, { + primary: true, + }); + rowsByName = new Map([...rowsByName, ...primaryRows]); } - return subdomainOwnerId !== appOwnerId; + + return apps.map((app, i) => { + const subdomain = subdomains[i]; + if (!subdomain) return false; + return backingRowIsUnavailable(app, rowsByName.get(subdomain) ?? null); + }); } /**