perf: batch per-app origin, canonical and hosted-backing lookups in app listings (PUT-1892) (#4013)

This commit is contained in:
Daniel Salazar authored and GitHub committed 2026-10-01 18:30:06 -07:00
1 parent 4f31388dad
commit e6bda950f0
12 files changed
+1628 -123

No files matched your search

+109 -49
View File
@@ -34,7 +34,7 @@ import {
buildHostedBackingDenial,
buildHostedSubdomainIndexUrlCandidates,
extractPuterHostedSubdomain,
hostedIndexUrlBackingIsUnavailable,
hostedIndexUrlBackingsAreUnavailable,
} from '../../util/hostedAppBacking.js';
import {
decodeCursor,
@@ -119,6 +119,19 @@ const hasIndexUrlUniquenessExemption = (candidates) => {
return false;
};
/** Parsed `protocol//hostname[:port]` origin of an index_url, or null. */
function indexUrlOrigin(indexUrl) {
if (!indexUrl) return null;
try {
const parsed = new URL(indexUrl);
return `${parsed.protocol}//${parsed.hostname}${
parsed.port ? `:${parsed.port}` : ''
}`;
} catch {
return null;
}
}
/**
* Driver exposing the `puter-apps` interface.
*
@@ -407,20 +420,34 @@ export class AppDriver extends PuterDriver {
// Pre-fetch in parallel:
// - per-uid stats (already pipelined inside getAppsStats)
// - filetype associations as a single IN-list query (was N queries)
const [statsByUid, filetypesByAppId] = await Promise.all([
// - canonical-index-url resolution and the hosted-backing check,
// each batched across every visible app
const [
statsByUid,
filetypesByAppId,
canonicalByApp,
hostedUnavailableByApp,
] = await Promise.all([
this.appStore.getAppsStats(visible.map((a) => a.uid)),
this.appStore.getFiletypeAssociationsByIds(
visible.map((a) => a.id),
),
this.#resolveCanonicalForIndexUrls(visible),
this.#hostedBackingUnavailableFlags(visible),
]);
const items = await Promise.all(
visible.map((app) =>
this.#toClient(app, actor, {
...params,
stats: statsByUid.get(app.uid),
filetypes: filetypesByAppId.get(app.id) ?? [],
}),
visible.map((app, i) =>
this.#toClient(
app,
actor,
{ ...params, stats: statsByUid.get(app.uid) },
{
filetypes: filetypesByAppId.get(app.id) ?? [],
canonical: canonicalByApp[i],
hostedBackingUnavailable: hostedUnavailableByApp[i],
},
),
),
);
if (!paginated) return items;
@@ -891,9 +918,9 @@ export class AppDriver extends PuterDriver {
// -- Serialization ------------------------------------------------
/**
* Resolve the canonical app row that backs `app.index_url`.
*
* Returns `{ origin, expectedUid, canonicalApp }`:
* Resolve the canonical app row backing each app's `index_url`, batched
* across `apps`. Returns an array aligned with `apps`, each entry `{
* origin, expectedUid, canonicalApp }`:
*
* - `origin` — the parsed origin string from `index_url`.
* - `expectedUid` — the canonical app uid for that origin (oldest
@@ -913,61 +940,94 @@ export class AppDriver extends PuterDriver {
* pre-existing data from before the `subdomain_not_owned` check leaks
* the victim's index_url.
*
* Returns `null` when there's no `index_url` or it doesn't parse.
* An entry is `null` when its app has no `index_url` or it doesn't parse.
*/
async #resolveCanonicalForIndexUrl(app) {
if (!app.index_url) return null;
let origin;
try {
const parsed = new URL(app.index_url);
origin = `${parsed.protocol}//${parsed.hostname}${
parsed.port ? `:${parsed.port}` : ''
}`;
} catch {
return null;
async #resolveCanonicalForIndexUrls(apps) {
const origins = apps.map((app) => indexUrlOrigin(app.index_url));
const uniqueOrigins = [...new Set(origins.filter((o) => o !== null))];
let uidByOrigin = new Map();
if (uniqueOrigins.length > 0) {
try {
uidByOrigin =
await this.services.auth.appUidsFromOrigins(uniqueOrigins);
} catch {
uidByOrigin = new Map();
}
}
try {
const expectedUid =
await this.services.auth.appUidFromOrigin(origin);
// Avoid a needless DB hit on the self-match common case —
// `app` is already the row we'd be re-fetching.
const canonicalApp =
expectedUid && expectedUid !== app.uid
? await this.appStore.getByUid(expectedUid)
: app;
return { origin, expectedUid, canonicalApp };
} catch {
return null;
// Avoid a needless DB hit on the self-match common case — `app` is
// already the row we'd be re-fetching.
const uidsToFetch = new Set();
for (let i = 0; i < apps.length; i++) {
if (!origins[i]) continue;
const expectedUid = uidByOrigin.get(origins[i]) ?? null;
if (expectedUid && expectedUid !== apps[i].uid) {
uidsToFetch.add(expectedUid);
}
}
let canonicalAppByUid = new Map();
if (uidsToFetch.size > 0) {
try {
canonicalAppByUid = await this.appStore.getByUids([
...uidsToFetch,
]);
} catch {
canonicalAppByUid = new Map();
}
}
return apps.map((app, i) => {
const origin = origins[i];
if (!origin) return null;
const expectedUid = uidByOrigin.get(origin) ?? null;
if (!expectedUid) return null;
if (expectedUid === app.uid) {
return { origin, expectedUid, canonicalApp: app };
}
return {
origin,
expectedUid,
canonicalApp: canonicalAppByUid.get(expectedUid) ?? null,
};
});
}
/**
* Launch-safety check for puter-hosted `index_url`s. See
* `util/hostedAppBacking.ts` — the check lives there because every producer
* of launchable app metadata needs it, not just this driver.
* Launch-safety check for puter-hosted `index_url`s, batched across `apps`.
* See `util/hostedAppBacking.ts` — the check lives there because every
* producer of launchable app metadata needs it, not just this driver.
*/
async #hostedIndexUrlBackingIsUnavailable(app) {
return hostedIndexUrlBackingIsUnavailable({
app,
async #hostedBackingUnavailableFlags(apps) {
return hostedIndexUrlBackingsAreUnavailable({
apps,
subdomainStore: this.stores.subdomain,
config: this.config,
});
}
async #toClient(app, actor, params = {}) {
async #toClient(app, actor, params = {}, prefetched = {}) {
if (!app) return null;
// `select` pre-fetches filetypes for every visible app in one
// batched query and threads them through `params.filetypes` to
// avoid the N+1 in this hot loop. Single-app callers (`read`,
// `create`, `update`) fall back to the per-app query.
// `select` batches these lookups per page and passes them in
// `prefetched`; single-app callers resolve them here. Never read them
// from `params`, which carries the RPC caller's input.
const [filetypes, canonicalForIndexUrl, hostedBackingUnavailable] =
await Promise.all([
params.filetypes !== undefined
? Promise.resolve(params.filetypes)
prefetched.filetypes !== undefined
? Promise.resolve(prefetched.filetypes)
: this.appStore.getFiletypeAssociations(app.id),
this.#resolveCanonicalForIndexUrl(app),
this.#hostedIndexUrlBackingIsUnavailable(app),
prefetched.canonical !== undefined
? Promise.resolve(prefetched.canonical)
: this.#resolveCanonicalForIndexUrls([app]).then(
(r) => r[0],
),
prefetched.hostedBackingUnavailable !== undefined
? Promise.resolve(prefetched.hostedBackingUnavailable)
: this.#hostedBackingUnavailableFlags([app]).then(
(r) => r[0],
),
]);
const createdFromOrigin =
+355
View File
@@ -1983,3 +1983,358 @@ describe('AppDriver hosted-subdomain ownership check', () => {
).not.toBe(false);
});
});
// -- select: batched #toClient query counts --
//
// Canonical index_url resolution and the hosted-backing check are batched per
// page, so these counts must stay flat as the page grows.
describe('AppDriver.select query-count regression', () => {
const makeMixedApps = async (count: number) => {
const { actor, userId } = await makeUser();
for (let i = 0; i < count; i++) {
const kind = i % 3;
if (kind === 0) {
// Live hosted: owns the subdomain it points at.
const sub = uniqueName(`qclive${i}`);
await server.stores.subdomain.create({ userId, subdomain: sub });
await withActor(actor, () =>
driver.create({
object: {
name: uniqueName(`qc-live-${i}`),
title: 't',
index_url: `https://${sub}.site.puter.localhost/`,
},
}),
);
} else if (kind === 1) {
// Dangling hosted: the subdomain is gone by the time we read.
const sub = uniqueName(`qcdang${i}`);
const row = await server.stores.subdomain.create({
userId,
subdomain: sub,
});
await withActor(actor, () =>
driver.create({
object: {
name: uniqueName(`qc-dangling-${i}`),
title: 't',
index_url: `https://${sub}.site.puter.localhost/`,
},
}),
);
await server.stores.subdomain.deleteByUuid(
String((row as { uuid: string }).uuid),
{ userId },
);
} else {
// External, with a path — not puter-hosted at all.
await withActor(actor, () =>
driver.create({
object: {
name: uniqueName(`qc-ext-${i}`),
title: 't',
index_url: `${uniqueIndexUrl()}some/path`,
},
}),
);
}
}
return actor;
};
const countQueriesForSelect = async (actor: Actor) => {
const read = vi.spyOn(server.clients.db, 'read');
const pread = vi.spyOn(server.clients.db, 'pread');
try {
const items = (await withActor(actor, () =>
driver.select({ predicate: ['user-can-edit'] }),
)) as Array<Record<string, unknown>>;
const counts = { indexUrlIn: 0, subdomains: 0, appsUidEq: 0 };
for (const call of read.mock.calls) {
const sql = call[0] as string;
if (/`index_url`\s+IN\s*\(/i.test(sql)) {
counts.indexUrlIn++;
} else if (sql.includes('`subdomains`')) {
counts.subdomains++;
} else if (
sql.includes('`apps`') &&
/`uid`\s*=\s*\?/.test(sql)
) {
counts.appsUidEq++;
}
}
return {
counts,
pread: pread.mock.calls.length,
itemCount: items.length,
};
} finally {
read.mockRestore();
pread.mockRestore();
}
};
it('keeps index_url / subdomains / per-app-uid query counts constant from 3 apps to 30 apps', async () => {
const actor3 = await makeMixedApps(3);
const actor30 = await makeMixedApps(30);
const small = await countQueriesForSelect(actor3);
const large = await countQueriesForSelect(actor30);
expect(small.itemCount).toBe(3);
expect(large.itemCount).toBe(30);
// Sanity check the classifier actually saw the shapes it's counting.
expect(small.counts.indexUrlIn).toBeGreaterThan(0);
expect(small.counts.subdomains).toBeGreaterThan(0);
expect(large.counts).toEqual(small.counts);
expect(large.pread).toBe(small.pread);
});
});
// -- select / read parity --
describe('AppDriver.select / read parity', () => {
const findViaBroadSelect = async (
actor: Actor,
uid: string,
): Promise<Record<string, unknown> | undefined> => {
let cursor: string | null | undefined = null;
do {
const page = (await withActor(actor, () =>
driver.select({ limit: 50, cursor }),
)) as { items: Array<Record<string, unknown>>; cursor?: string };
const found = page.items.find((r) => r.uid === uid);
if (found) return found;
cursor = page.cursor;
} while (cursor);
return undefined;
};
it('every select item (minus stats) deep-equals the corresponding read', async () => {
const { actor } = await makeUser();
const names: string[] = [];
for (let i = 0; i < 4; i++) {
const name = uniqueName(`parity${i}`);
names.push(name);
await withActor(actor, () =>
driver.create({
object: { name, title: 't', index_url: uniqueIndexUrl() },
}),
);
}
const selectResult = (await withActor(actor, () =>
driver.select({ predicate: ['user-can-edit'] }),
)) as Array<Record<string, unknown>>;
const ours = selectResult.filter((item) =>
names.includes(item.name as string),
);
expect(ours.length).toBe(names.length);
for (const item of ours) {
const read = await withActor(actor, () =>
driver.read({ uid: item.uid as string }),
);
const { stats: _itemStats, ...itemRest } = item;
const { stats: _readStats, ...readRest } = read;
expect(itemRest).toEqual(readRest);
}
});
it('sets created_from_origin on the canonical hosted row and null on a duplicate', async () => {
const { actor, userId } = await makeUser();
const sub = uniqueName('cfo');
await server.stores.subdomain.create({ userId, subdomain: sub });
const url = `https://${sub}.site.puter.localhost/`;
const canonical = await withActor(actor, () =>
driver.create({
object: {
name: uniqueName('cfo-canon'),
title: 't',
index_url: url,
},
}),
);
// A duplicate row at the same index_url — `create` would normally
// refuse this; direct insert mirrors the pre-existing-data shape
// the canonical resolver has to cope with.
const dupUid = `app-${uuidv4()}`;
await server.clients.db.write(
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
[dupUid, uniqueName('cfo-dup'), 'dup', url, userId],
);
const result = (await withActor(actor, () =>
driver.select({ predicate: ['user-can-edit'] }),
)) as Array<Record<string, unknown>>;
const canonItem = result.find((r) => r.uid === canonical.uid);
const dupItem = result.find((r) => r.uid === dupUid);
expect(canonItem?.created_from_origin).toBe(
`https://${sub}.site.puter.localhost`,
);
expect(dupItem?.created_from_origin).toBeNull();
});
it('gates the canonical-private row: a public duplicate withholds index_url and denies access', async () => {
const ownerA = await makeUser();
const ownerB = await makeUser();
const sharedUrl = uniqueIndexUrl();
const uidA = `app-${uuidv4()}`;
const uidB = `app-${uuidv4()}`;
await server.clients.db.write(
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`, `is_private`) VALUES (?, ?, ?, ?, ?, ?)',
[
uidA,
uniqueName('priv-a'),
'Private A',
sharedUrl,
ownerA.userId,
1,
],
);
await server.clients.db.write(
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`, `is_private`) VALUES (?, ?, ?, ?, ?, ?)',
[
uidB,
uniqueName('pub-b'),
'Public B',
sharedUrl,
ownerB.userId,
0,
],
);
// B's own owner is not A's owner, so the gate still applies to them.
const result = (await withActor(ownerB.actor, () =>
driver.select({ predicate: ['user-can-edit'] }),
)) as Array<Record<string, unknown>>;
const bItem = result.find((r) => r.uid === uidB);
expect(bItem).toBeTruthy();
expect(bItem!.index_url).toBeUndefined();
expect(
(bItem!.privateAccess as { hasAccess?: boolean }).hasAccess,
).toBe(false);
});
it('dangling hosted app via select: owner keeps index_url, others do not', async () => {
const owner = await makeUser();
const other = await makeUser();
const sub = uniqueName('qcdangsel');
const row = await server.stores.subdomain.create({
userId: owner.userId,
subdomain: sub,
});
const created = await withActor(owner.actor, () =>
driver.create({
object: {
name: uniqueName('dangling-sel'),
title: 't',
index_url: `https://${sub}.site.puter.localhost/`,
},
}),
);
await server.stores.subdomain.deleteByUuid(
String((row as { uuid: string }).uuid),
{ userId: owner.userId },
);
const ownerResult = (await withActor(owner.actor, () =>
driver.select({ predicate: ['user-can-edit'] }),
)) as Array<Record<string, unknown>>;
const ownerItem = ownerResult.find((r) => r.uid === created.uid);
expect(ownerItem?.index_url).toBe(created.index_url);
expect(
(ownerItem?.privateAccess as { reason?: string } | undefined)
?.reason,
).toBe('hosted_backing_unavailable');
const otherItem = await findViaBroadSelect(
other.actor,
created.uid as string,
);
expect(otherItem?.index_url).toBeUndefined();
expect(
(otherItem?.privateAccess as { hasAccess?: boolean } | undefined)
?.hasAccess,
).toBe(false);
});
it('a blocked origin resolves created_from_origin to null without select throwing', async () => {
const { actor } = await makeUser();
const blockedHost = `blocked-${Math.random().toString(36).slice(2, 10)}.test`;
const created = await withActor(actor, () =>
driver.create({
object: {
name: uniqueName('blocked-sel'),
title: 't',
index_url: `https://${blockedHost}/app`,
},
}),
);
await server.clients.db.write(
'INSERT INTO `blocked_app_origins` (`domain`, `include_subdomains`) VALUES (?, ?)',
[blockedHost, 0],
);
(
server.services.appOriginBlocklist as { invalidate: () => void }
).invalidate();
const result = (await withActor(actor, () =>
driver.select({ predicate: ['user-can-edit'] }),
)) as Array<Record<string, unknown>>;
const item = result.find((r) => r.uid === created.uid);
expect(item).toBeTruthy();
expect(item!.created_from_origin).toBeNull();
});
});
// -- read: prefetched-shaped params cannot be spoofed --
describe('AppDriver.read prefetched-param isolation', () => {
it('ignores caller-supplied hostedBackingUnavailable/canonical/filetypes on a dangling app', async () => {
const owner = await makeUser();
const attacker = await makeUser();
const sub = uniqueName('sec-dangling');
const row = await server.stores.subdomain.create({
userId: owner.userId,
subdomain: sub,
});
const created = await withActor(owner.actor, () =>
driver.create({
object: {
name: uniqueName('sec-app'),
title: 't',
index_url: `https://${sub}.site.puter.localhost/`,
filetype_associations: ['.puter'],
},
}),
);
await server.stores.subdomain.deleteByUuid(
String((row as { uuid: string }).uuid),
{ userId: owner.userId },
);
const result = await withActor(attacker.actor, () =>
driver.read({
uid: created.uid,
params: {
hostedBackingUnavailable: false,
canonical: null,
filetypes: ['x'],
},
}),
);
// The spoofed `hostedBackingUnavailable: false` doesn't suppress the
// real denial, so the dangling app's index_url is still withheld.
expect(result.index_url).toBeUndefined();
// The spoofed `filetypes` doesn't override the DB-backed list.
expect(result.filetype_associations).toEqual(
expect.arrayContaining(['puter']),
);
expect(result.filetype_associations).not.toContain('x');
});
});
@@ -1074,3 +1074,48 @@ describe('SubdomainDriver associated_app derivation', () => {
expect(read.associated_app).toBeNull();
});
});
// -- associated_app derivation at scale --
//
// ~24 index_url candidates per subdomain: a page this size must be chunked
// to stay under SQLite's bound-parameter limit.
describe('SubdomainDriver.select at scale', () => {
it('resolves associated_app for 1,400 subdomains without tripping the SQL variable limit', async () => {
const { actor, userId } = await makeUser();
const prefix = `scale-${Math.random().toString(36).slice(2, 8)}-`;
const count = 1400;
const names: string[] = [];
for (let i = 0; i < count; i++) {
names.push(`${prefix}${i}`);
}
// Bulk-insert directly — bypasses the driver's per-user quota, the
// same way a bulk-provisioned fleet of sites would exist in prod.
const placeholders = names.map(() => '(?, ?, ?)').join(', ');
const values: unknown[] = [];
for (const name of names) {
values.push(uuidv4(), name, userId);
}
await server.clients.db.write(
`INSERT INTO \`subdomains\` (\`uuid\`, \`subdomain\`, \`user_id\`) VALUES ${placeholders}`,
values,
);
const lastSub = names[names.length - 1]!;
const app = await createAppWithIndexUrl(
userId,
`http://${lastSub}.site.puter.localhost/`,
);
const result = (await withActor(actor, () =>
driver.select({}),
)) as Array<Record<string, unknown>>;
expect(result.length).toBe(count);
const lastItem = result.find((r) => r.subdomain === lastSub);
expect(
(lastItem?.associated_app as { uid: string } | null)?.uid,
).toBe(app.uid);
}, 30_000);
});
@@ -188,10 +188,9 @@ export class SubdomainDriver extends PuterDriver {
// they have open is not them.
// See `util/hostedAppBacking.ts` for the wider rule.
//
// Last check before the insert on purpose: `apps.index_url` is
// unindexed, so this scan only runs for a request that would otherwise
// have created the row, and it stays behind the same root_dir gate as
// the existing uniqueness answer.
// Last check before the insert on purpose: it stays behind the same
// root_dir gate as the existing uniqueness answer, so it only runs
// for a request that would otherwise have created the row.
const appsHoldingName = (await this.stores.app.listByIndexUrlCandidates(
buildHostedSubdomainIndexUrlCandidates(subdomain, this.config),
)) as Array<Record<string, unknown>>;
@@ -737,7 +736,8 @@ export class SubdomainDriver extends PuterDriver {
* hosting domains × protocols × paths). Returns a `rowUuid → appId` map.
* Rows with no matching app are absent.
*
* Runs one batched DB query regardless of input size.
* Goes through the batched `listByIndexUrlCandidates` store lookup rather
* than one query per row, regardless of input size.
*/
async #deriveAssociatedAppIdByRowUuid(
rows: Array<Record<string, unknown>>,
@@ -782,16 +782,14 @@ export class SubdomainDriver extends PuterDriver {
return result;
}
const userIds = [...userIdToRowMeta.keys()];
const userPlaceholders = userIds.map(() => '?').join(', ');
const candidateList = [...allCandidates];
const urlPlaceholders = candidateList.map(() => '?').join(', ');
const matches = (await this.clients.db.read(
`SELECT \`id\`, \`owner_user_id\`, \`index_url\` FROM \`apps\`
WHERE \`owner_user_id\` IN (${userPlaceholders})
AND \`index_url\` IN (${urlPlaceholders})`,
[...userIds, ...candidateList],
const matches = (await this.stores.app.listByIndexUrlCandidates(
candidateList,
)) as Array<Record<string, unknown>>;
// Oldest row wins when more than one app matches the same candidate
// for the same owner — the owner-match filter below is what actually
// restricts matches to rows this batch cares about.
matches.sort((a, b) => Number(a.id) - Number(b.id));
for (const m of matches) {
const appId = typeof m.id === 'number' ? m.id : Number(m.id);
@@ -1811,6 +1811,155 @@ describe('AuthService (integration)', () => {
});
});
describe('appUidsFromOrigins', () => {
it('matches appUidFromOrigin across hosting variants, an unknown external, and a repointed origin', async () => {
const sub = `batch-${Math.random().toString(36).slice(2, 10)}`;
const origins = [
`https://${sub}.site.puter.localhost`,
`https://${sub}.host.puter.localhost`,
`http://${sub}.app.puter.localhost`,
`https://external-${uuidv4()}.example.com`,
`chrome-extension://${uuidv4()}`,
];
const repointOrigin = `https://repoint-batch-${uuidv4()}.example.com`;
const repointed = await server.stores.app.createFromOrigin(
await authService.appUidFromOrigin(repointOrigin),
authService.canonicalizeOrigin(repointOrigin),
);
await server.stores.app.update(repointed.id, {
index_url: `https://repoint-new-${uuidv4()}.example.com`,
});
origins.push(repointOrigin);
const expected = new Map<string, string>();
for (const origin of origins) {
expected.set(
origin,
await authService.appUidFromOrigin(origin),
);
}
const batched = await authService.appUidsFromOrigins(origins);
for (const origin of origins) {
expect(batched.get(origin)).toBe(expected.get(origin));
}
// Every hosting variant of the same subdomain collapses to one uid.
expect(batched.get(origins[0])).toBe(batched.get(origins[1]));
expect(batched.get(origins[0])).toBe(batched.get(origins[2]));
// The repointed origin moved on to a successor uid, not the
// stale row's.
expect(batched.get(repointOrigin)).not.toBe(repointed.uid);
});
it('resolves unparseable origins to null instead of throwing', async () => {
const result = await authService.appUidsFromOrigins([
'not-a-url',
'javascript:alert(document.domain)',
]);
expect(result.get('not-a-url')).toBeNull();
expect(result.get('javascript:alert(document.domain)')).toBeNull();
});
it('resolves a blocked origin to null without throwing or affecting other origins', async () => {
const blockedHost = `blocked-batch-${uuidv4()}.example.com`;
await server.clients.db.write(
'INSERT INTO `blocked_app_origins` (`domain`, `include_subdomains`) VALUES (?, ?)',
[blockedHost, 0],
);
(
server.services.appOriginBlocklist as { invalidate: () => void }
).invalidate();
const okOrigin = `https://ok-batch-${uuidv4()}.example.com`;
const expectedOk = await authService.appUidFromOrigin(okOrigin);
const result = await authService.appUidsFromOrigins([
`https://${blockedHost}`,
okOrigin,
]);
expect(result.get(`https://${blockedHost}`)).toBeNull();
expect(result.get(okOrigin)).toBe(expectedOk);
});
it('prefers a private row over an older public stub, breaking ties by lowest id, in one batch', async () => {
const user = await makeUser();
const sub = `batch-priv-${Math.random().toString(36).slice(2, 10)}`;
await server.stores.app.createFromOrigin(
`app-${uuidv4()}`,
`https://${sub}.host.puter.localhost`,
{ ownerUserId: user.id },
);
const realUid = `app-${uuidv4()}`;
await server.clients.db.write(
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`, `is_private`) VALUES (?, ?, ?, ?, ?, ?)',
[
realUid,
`real-${sub}`,
'Real app',
`https://${sub}.app.puter.localhost`,
user.id,
1,
],
);
const origin = `https://${sub}.host.puter.localhost`;
const result = await authService.appUidsFromOrigins([origin]);
expect(result.get(origin)).toBe(realUid);
});
it('a private row on one index_url variant wins over a lower-id public row on another, in both paths', async () => {
// Both rows match the same origin's candidate set but live under
// different exact `index_url` strings — the winners reduction has
// to look across every matching variant, not just one.
const base = `https://variant-${uuidv4()}.example.com`;
const publicUid = `app-${uuidv4()}`;
const privateUid = `app-${uuidv4()}`;
await server.clients.db.write(
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `is_private`) VALUES (?, ?, ?, ?, ?)',
[publicUid, `pub-${uuidv4()}`, 'Public', `${base}/`, 0],
);
await server.clients.db.write(
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `is_private`) VALUES (?, ?, ?, ?, ?)',
[
privateUid,
`priv-${uuidv4()}`,
'Private',
`${base}/index.html`,
1,
],
);
await expect(authService.appUidFromOrigin(base)).resolves.toBe(
privateUid,
);
const batched = await authService.appUidsFromOrigins([base]);
expect(batched.get(base)).toBe(privateUid);
});
it('does not cross-assign uids between origins that each have their own canonical row', async () => {
const subA = `batch-cross-a-${Math.random().toString(36).slice(2, 8)}`;
const subB = `batch-cross-b-${Math.random().toString(36).slice(2, 8)}`;
const originA = `https://${subA}.site.puter.localhost`;
const originB = `https://${subB}.site.puter.localhost`;
const appA = await server.stores.app.createFromOrigin(
await authService.appUidFromOrigin(originA),
authService.canonicalizeOrigin(originA),
);
const appB = await server.stores.app.createFromOrigin(
await authService.appUidFromOrigin(originB),
authService.canonicalizeOrigin(originB),
);
const result = await authService.appUidsFromOrigins([
originA,
originB,
]);
expect(result.get(originA)).toBe(appA.uid);
expect(result.get(originB)).toBe(appB.uid);
});
});
describe('subdomainOwnerIdFromOrigin', () => {
// Test servers inherit the four hosting domains (production:
// puter.site / puter.host / puter.app / puter.dev) from
@@ -2937,6 +3086,23 @@ describe('AuthService.appUidFromOrigin — aliased hosts', () => {
);
expect(withPort).not.toBe(withoutPort);
});
it('appUidsFromOrigins collapses an alias group onto one uid, batched', async () => {
const alphaOrigin = 'https://alpha.example.com';
const betaOrigin = 'https://beta.example.com';
const ungroupedOrigin = 'https://gamma.example.com';
const result = await authService.appUidsFromOrigins([
alphaOrigin,
betaOrigin,
ungroupedOrigin,
]);
expect(result.get(betaOrigin)).toBe(result.get(alphaOrigin));
expect(result.get(ungroupedOrigin)).not.toBe(result.get(alphaOrigin));
expect(result.get(alphaOrigin)).toBe(
await authService.appUidFromOrigin(alphaOrigin),
);
});
});
describe('AuthService.subdomainOwnerIdFromOrigin — edge cases', () => {
+215 -26
View File
@@ -819,6 +819,102 @@ export class AuthService extends PuterService {
return this.#derivedAppUidForOrigin(appOrigin);
}
/**
* Batched sibling of {@link appUidFromOrigin} for listing paths that need a
* uid per origin without a round trip each. Unparseable or blocked origins
* resolve to null instead of throwing.
*/
async appUidsFromOrigins(
origins: string[],
): Promise<Map<string, string | null>> {
const result = new Map<string, string | null>();
const uniqueOrigins = [...new Set(origins)];
const appOriginByOrigin = new Map<string, string | null>();
await Promise.all(
uniqueOrigins.map(async (origin) => {
try {
appOriginByOrigin.set(
origin,
await this.#appOriginFor(origin),
);
} catch {
appOriginByOrigin.set(origin, null);
}
}),
);
const uniqueAppOrigins = [
...new Set(
[...appOriginByOrigin.values()].filter(
(o): o is string => o !== null,
),
),
];
const blockedByAppOrigin = new Map<string, boolean>();
await Promise.all(
uniqueAppOrigins.map(async (appOrigin) => {
try {
const block =
await this.services.appOriginBlocklist.isOriginBlocked(
appOrigin,
);
blockedByAppOrigin.set(appOrigin, block.blocked);
} catch {
blockedByAppOrigin.set(appOrigin, true);
}
}),
);
const resolvableAppOrigins = uniqueAppOrigins.filter(
(appOrigin) => !blockedByAppOrigin.get(appOrigin),
);
const canonicalByAppOrigin =
await this.#findCanonicalAppUidsForOrigins(resolvableAppOrigins);
// Gen-0 derived uid per app origin that missed the canonical lookup,
// prefetched in one batch so `#derivedAppUidForOrigin`'s loop doesn't
// pay a round trip for the common (no repoint) case.
const missedAppOrigins = resolvableAppOrigins.filter(
(appOrigin) => !canonicalByAppOrigin.get(appOrigin),
);
const gen0ByAppOrigin = new Map<string, string>();
for (const appOrigin of missedAppOrigins) {
gen0ByAppOrigin.set(appOrigin, this.#originUid(appOrigin, 0));
}
const gen0Uids = [...new Set(gen0ByAppOrigin.values())];
const prefetched =
gen0Uids.length > 0
? await this.stores.app.getByUids(gen0Uids)
: new Map<string, { index_url?: unknown }>();
const derivedByAppOrigin = new Map<string, string>();
for (const appOrigin of missedAppOrigins) {
const gen0Uid = gen0ByAppOrigin.get(appOrigin)!;
const uid = await this.#derivedAppUidForOrigin(appOrigin, (uid) =>
uid === gen0Uid
? Promise.resolve(prefetched.get(uid) ?? null)
: this.stores.app.getByUid(uid),
);
derivedByAppOrigin.set(appOrigin, uid);
}
for (const origin of origins) {
const appOrigin = appOriginByOrigin.get(origin) ?? null;
if (appOrigin === null || blockedByAppOrigin.get(appOrigin)) {
result.set(origin, null);
continue;
}
const canonicalUid = canonicalByAppOrigin.get(appOrigin);
result.set(
origin,
canonicalUid ?? derivedByAppOrigin.get(appOrigin) ?? null,
);
}
return result;
}
/**
* Normalized origin of `url` with aliased hosts and hosting-domain variants
* collapsed, after `app.from-origin` listeners have rewritten it. Null when
@@ -834,17 +930,29 @@ export class AuthService extends PuterService {
return event.origin;
}
/** `app-<uuidv5(origin)>` for generation `gen` (0 is the first-visit uid). */
#originUid(origin: string, gen: number): string {
const name = gen === 0 ? origin : `${origin}#${gen}`;
return `app-${uuidv5(name, APP_ORIGIN_UUID_NAMESPACE)}`;
}
/**
* `app-<uuidv5(origin)>`, unless that uid's row now lives at another
* origin. A repointed row keeps its uid (and the data and grants keyed to
* it), so the origin it left moves on to a successor uid instead of
* resolving to someone's app it no longer serves.
* resolving to someone's app it no longer serves. `getByUid` is injectable
* so a batch caller can serve prefetched rows.
*/
async #derivedAppUidForOrigin(origin: string): Promise<string> {
async #derivedAppUidForOrigin(
origin: string,
getByUid: (
uid: string,
) => Promise<{ index_url?: unknown } | null | undefined> = (uid) =>
this.stores.app.getByUid(uid),
): Promise<string> {
for (let gen = 0; gen <= MAX_ORIGIN_UID_GENERATIONS; gen++) {
const name = gen === 0 ? origin : `${origin}#${gen}`;
const uid = `app-${uuidv5(name, APP_ORIGIN_UUID_NAMESPACE)}`;
const app = await this.stores.app.getByUid(uid);
const uid = this.#originUid(origin, gen);
const app = await getByUid(uid);
if (!app) return uid;
if (
typeof app.index_url === 'string' &&
@@ -1058,21 +1166,17 @@ export class AuthService extends PuterService {
}
/**
* Find the real app row whose `index_url` canonically matches `origin`.
*
* Build candidate URLs from the origin's subdomain crossed with every
* configured hosting domain (static + private, with and without ports).
* Prefer private rows, then the oldest match, for deterministic
* tie-breaking across historically-duplicated rows.
* Every `index_url` string that would canonically match `origin` — the
* origin's subdomain crossed with every configured hosting domain (static
* and private, with and without ports), alias-group hosts, and protocol
* variants.
*/
async #findCanonicalAppUidForOrigin(
origin: string,
): Promise<string | null> {
#indexUrlCandidatesForOrigin(origin: string): string[] {
let parsed: URL;
try {
parsed = new URL(origin);
} catch {
return null;
return [];
}
const config = this.config as { protocol?: string };
@@ -1120,18 +1224,103 @@ export class AuthService extends PuterService {
urlCandidates.push(base, `${base}/`, `${base}/index.html`);
}
}
const uniqueCandidates = [...new Set(urlCandidates)];
if (uniqueCandidates.length === 0) return null;
return [...new Set(urlCandidates)];
}
const placeholders = uniqueCandidates.map(() => '?').join(', ');
// Private rows win over public duplicates; oldest id breaks ties.
const rows = (await this.clients.db.read(
`SELECT \`uid\` FROM \`apps\` WHERE \`index_url\` IN (${placeholders}) ` +
`ORDER BY CASE WHEN \`is_private\` = ${this.clients.db.booleanLiteral(true)} THEN 0 ELSE 1 END, \`id\` ASC LIMIT 1`,
uniqueCandidates,
)) as Array<{ uid?: string }>;
const uid = rows[0]?.uid;
return typeof uid === 'string' && uid ? uid : null;
/**
* Uid of the real app row whose `index_url` canonically matches `origin`;
* private rows first, then the oldest, across historical duplicates.
*/
async #findCanonicalAppUidForOrigin(
origin: string,
): Promise<string | null> {
return this.stores.app.findCanonicalUidByIndexUrlCandidates(
this.#indexUrlCandidatesForOrigin(origin),
);
}
/**
* Batched {@link #findCanonicalAppUidForOrigin}, keyed by the origins passed
* in, with the same private-first, oldest-id rule.
*/
async #findCanonicalAppUidsForOrigins(
origins: string[],
): Promise<Map<string, string | null>> {
const result = new Map<string, string | null>();
const candidateSetByOrigin = new Map<string, Set<string>>();
const allCandidates = new Set<string>();
for (const origin of origins) {
const candidates = this.#indexUrlCandidatesForOrigin(origin);
candidateSetByOrigin.set(
origin,
new Set(candidates.map((c) => c.toLowerCase())),
);
for (const c of candidates) allCandidates.add(c);
}
if (allCandidates.size === 0) {
for (const origin of origins) result.set(origin, null);
return result;
}
const winners = (await this.stores.app.listIndexUrlWinners([
...allCandidates,
])) as Array<{
index_url: unknown;
private_id: unknown;
min_id: unknown;
}>;
const winnerByIndexUrl = new Map<string, (typeof winners)[number]>();
for (const winner of winners) {
if (typeof winner.index_url === 'string') {
winnerByIndexUrl.set(winner.index_url.toLowerCase(), winner);
}
}
// Lowest private id across the origin's matching groups, else lowest id.
const winnerIdByOrigin = new Map<string, number>();
for (const origin of origins) {
const candidateSet = candidateSetByOrigin.get(origin);
if (!candidateSet) continue;
let bestPrivateId: number | null = null;
let bestMinId: number | null = null;
for (const candidate of candidateSet) {
const winner = winnerByIndexUrl.get(candidate);
if (!winner) continue;
if (winner.private_id != null) {
const id = Number(winner.private_id);
if (bestPrivateId === null || id < bestPrivateId) {
bestPrivateId = id;
}
}
if (winner.min_id != null) {
const id = Number(winner.min_id);
if (bestMinId === null || id < bestMinId) {
bestMinId = id;
}
}
}
const winnerId = bestPrivateId ?? bestMinId;
if (winnerId !== null) winnerIdByOrigin.set(origin, winnerId);
}
const uniqueIds = [...new Set(winnerIdByOrigin.values())];
// Aggregates can come back as strings; they were Number()-ed above to
// match `getByIds` keys.
const appsById =
uniqueIds.length > 0
? await this.stores.app.getByIds(uniqueIds)
: new Map<number, { uid?: unknown }>();
for (const origin of origins) {
const id = winnerIdByOrigin.get(origin);
const uid = id !== undefined ? appsById.get(id)?.uid : undefined;
result.set(origin, typeof uid === 'string' && uid ? uid : null);
}
return result;
}
async getUserAppToken(actor: Actor, appUid: string): Promise<string> {
+98 -6
View File
@@ -58,6 +58,8 @@ const OLD_APP_NAME_TTL_MONTHS = 3;
// limit is 999; staying well under that keeps `getByIds` portable across
// backends without splitting the cap by driver.
const BULK_QUERY_CHUNK_SIZE = 200;
// Placeholders per index_url `IN (…)` chunk; under SQLite's old 999 default.
const INDEX_URL_CHUNK_SIZE = 900;
// Top-level all-time open/user counts: hot path, slow to compute. Cached
// lazily on read (pipelined MGET on every app list/read; misses query the
@@ -299,15 +301,105 @@ export class AppStore extends PuterStore {
/**
* Every app whose `index_url` matches one of `candidates`, with the owner
* and the app that built it. Used by the subdomain driver to decide who may
* re-create a hosted name that apps still point at.
* re-create a hosted name that apps still point at, and to derive a
* subdomain row's associated app.
*/
async listByIndexUrlCandidates(candidates) {
if (!Array.isArray(candidates) || candidates.length === 0) return [];
const placeholders = candidates.map(() => '?').join(', ');
return this.clients.db.read(
`SELECT \`id\`, \`uid\`, \`owner_user_id\`, \`app_owner\` FROM \`apps\` WHERE \`index_url\` IN (${placeholders})`,
[...candidates],
const uniqueCandidates = [
...new Set(
(Array.isArray(candidates) ? candidates : []).filter(
(c) => typeof c === 'string' && c.length > 0,
),
),
];
if (uniqueCandidates.length === 0) return [];
const rowsById = new Map();
for (
let offset = 0;
offset < uniqueCandidates.length;
offset += INDEX_URL_CHUNK_SIZE
) {
const chunk = uniqueCandidates.slice(
offset,
offset + INDEX_URL_CHUNK_SIZE,
);
const placeholders = chunk.map(() => '?').join(', ');
const rows = await this.clients.db.read(
`SELECT \`id\`, \`uid\`, \`owner_user_id\`, \`app_owner\`, \`index_url\` FROM \`apps\` WHERE \`index_url\` IN (${placeholders})`,
chunk,
);
for (const row of rows) rowsById.set(row.id, row);
}
return [...rowsById.values()];
}
/**
* Canonical app uid among rows matching `candidates`: private first, then
* oldest.
*/
async findCanonicalUidByIndexUrlCandidates(candidates) {
const uniqueCandidates = [
...new Set(
(Array.isArray(candidates) ? candidates : []).filter(
(c) => typeof c === 'string' && c.length > 0,
),
),
];
if (uniqueCandidates.length === 0) return null;
const placeholders = uniqueCandidates.map(() => '?').join(', ');
const rows = await this.clients.db.read(
`SELECT \`uid\` FROM \`apps\` WHERE \`index_url\` IN (${placeholders}) ` +
`ORDER BY CASE WHEN \`is_private\` = ${this.clients.db.booleanLiteral(true)} THEN 0 ELSE 1 END, \`id\` ASC LIMIT 1`,
uniqueCandidates,
);
const uid = rows[0]?.uid;
return typeof uid === 'string' && uid ? uid : null;
}
/**
* One row per distinct `index_url` in `candidates` with its lowest private
* id (`private_id`, null if none) and lowest id (`min_id`). An external
* index_url can be shared by thousands of apps, so batch callers pick
* canonical winners from these instead of every matching row.
*/
async listIndexUrlWinners(candidates) {
const uniqueCandidates = [
...new Set(
(Array.isArray(candidates) ? candidates : []).filter(
(c) => typeof c === 'string' && c.length > 0,
),
),
];
if (uniqueCandidates.length === 0) return [];
const winnersByIndexUrl = new Map();
for (
let offset = 0;
offset < uniqueCandidates.length;
offset += INDEX_URL_CHUNK_SIZE
) {
const chunk = uniqueCandidates.slice(
offset,
offset + INDEX_URL_CHUNK_SIZE,
);
const placeholders = chunk.map(() => '?').join(', ');
const rows = await this.clients.db.read(
`SELECT \`index_url\`, ` +
`MIN(CASE WHEN \`is_private\` = ${this.clients.db.booleanLiteral(true)} THEN \`id\` END) AS private_id, ` +
`MIN(\`id\`) AS min_id ` +
`FROM \`apps\` WHERE \`index_url\` IN (${placeholders}) GROUP BY \`index_url\``,
chunk,
);
// Deduped candidates land in one chunk each, so groups never span chunks.
for (const row of rows) {
if (typeof row.index_url === 'string') {
winnersByIndexUrl.set(row.index_url, row);
}
}
}
return [...winnersByIndexUrl.values()];
}
/**
+165
View File
@@ -1419,3 +1419,168 @@ describe('AppStore deleted-row tombstones', () => {
expect(await redis.get(`apps:uid:${uid}`)).not.toBeNull();
});
});
describe('AppStore listByIndexUrlCandidates', () => {
let server;
let appStore;
beforeAll(async () => {
server = await setupTestServer();
appStore = server.stores.app;
// Shared mock redis persists `apps:id:*` across servers while each
// fresh sqlite db restarts its id sequence at 1 — without this a
// freshly-created row can read back a stale cached row at the same id.
await clearAppCache(server.clients.redis);
});
afterAll(async () => {
await server?.shutdown();
});
const createApp = async (indexUrl) => {
const name = `idxurl-${Math.random().toString(36).slice(2, 10)}`;
return appStore.create(
{ name, title: name, index_url: indexUrl },
{ ownerUserId: 1 },
);
};
it('chunks a candidate list spanning more than one page and still finds the real URL at the end', async () => {
const real = await createApp('https://real-target.example.com/');
// 2,000 candidates is more than one 900-wide chunk; the real URL
// sits past the first chunk boundary.
const candidates = [];
for (let i = 0; i < 2000; i++) {
candidates.push(`https://decoy-${i}.example.com/`);
}
candidates.push(real.index_url);
const rows = await appStore.listByIndexUrlCandidates(candidates);
expect(rows.map((r) => r.id)).toEqual([real.id]);
});
it('dedupes a candidate repeated across chunk boundaries to one row', async () => {
const real = await createApp('https://dup-target.example.com/');
const candidates = Array.from({ length: 2000 }, () => real.index_url);
const rows = await appStore.listByIndexUrlCandidates(candidates);
expect(rows.length).toBe(1);
expect(rows[0].id).toBe(real.id);
});
it('returns [] for an empty or non-array candidate list', async () => {
expect(await appStore.listByIndexUrlCandidates([])).toEqual([]);
expect(await appStore.listByIndexUrlCandidates(null)).toEqual([]);
});
});
describe('AppStore findCanonicalUidByIndexUrlCandidates', () => {
let server;
let appStore;
beforeAll(async () => {
server = await setupTestServer();
appStore = server.stores.app;
await clearAppCache(server.clients.redis);
});
afterAll(async () => {
await server?.shutdown();
});
it('prefers the private row, then the oldest match', async () => {
const url = `https://canon-${Math.random().toString(36).slice(2, 10)}.example.com/`;
const pub = await appStore.create(
{ name: `pub-${Math.random().toString(36).slice(2, 8)}`, title: 't', index_url: url },
{ ownerUserId: 1 },
);
const privUid = `app-${Math.random().toString(36).slice(2, 10)}`;
await server.clients.db.write(
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `is_private`) VALUES (?, ?, ?, ?, ?)',
[privUid, `priv-${Math.random().toString(36).slice(2, 8)}`, 't', url, 1],
);
const uid = await appStore.findCanonicalUidByIndexUrlCandidates([url]);
expect(uid).toBe(privUid);
expect(uid).not.toBe(pub.uid);
});
it('returns null for an empty, non-array, or non-matching candidate list', async () => {
expect(await appStore.findCanonicalUidByIndexUrlCandidates([])).toBeNull();
expect(await appStore.findCanonicalUidByIndexUrlCandidates(null)).toBeNull();
expect(
await appStore.findCanonicalUidByIndexUrlCandidates([
'https://nothing-here.example.com/',
]),
).toBeNull();
});
});
describe('AppStore listIndexUrlWinners', () => {
let server;
let appStore;
beforeAll(async () => {
server = await setupTestServer();
appStore = server.stores.app;
await clearAppCache(server.clients.redis);
});
afterAll(async () => {
await server?.shutdown();
});
const createApp = async (indexUrl) => {
const name = `winner-${Math.random().toString(36).slice(2, 10)}`;
return appStore.create(
{ name, title: name, index_url: indexUrl },
{ ownerUserId: 1 },
);
};
it('collapses many apps sharing one index_url into a single winner row', async () => {
// An external dev-server default — the exact shape that can be
// shared by thousands of apps and would blow up a row-per-app query.
const sharedUrl = `http://localhost:${5000 + Math.floor(Math.random() * 1000)}/`;
for (let i = 0; i < 50; i++) {
await createApp(sharedUrl);
}
const otherUrl = `https://distinct-${Math.random().toString(36).slice(2, 10)}.example.com/`;
await createApp(otherUrl);
const rows = await appStore.listIndexUrlWinners([sharedUrl, otherUrl]);
expect(rows.length).toBe(2);
// One row per distinct index_url, never one per matching app.
expect(rows.length).toBeLessThanOrEqual(2);
const shared = rows.find((r) => r.index_url === sharedUrl);
expect(shared).toBeTruthy();
expect(shared.private_id == null).toBe(true);
expect(Number(shared.min_id)).toBeGreaterThan(0);
});
it('reports the lowest private id and the lowest id overall per index_url', async () => {
const url = `https://winner-${Math.random().toString(36).slice(2, 10)}.example.com/`;
const pub = await createApp(url);
const priv1Uid = `app-${Math.random().toString(36).slice(2, 10)}`;
await server.clients.db.write(
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `is_private`) VALUES (?, ?, ?, ?, ?)',
[priv1Uid, `priv1-${Math.random().toString(36).slice(2, 8)}`, 't', url, 1],
);
const priv1 = await appStore.getByUid(priv1Uid);
const priv2Uid = `app-${Math.random().toString(36).slice(2, 10)}`;
await server.clients.db.write(
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `is_private`) VALUES (?, ?, ?, ?, ?)',
[priv2Uid, `priv2-${Math.random().toString(36).slice(2, 8)}`, 't', url, 1],
);
const priv2 = await appStore.getByUid(priv2Uid);
const [winner] = await appStore.listIndexUrlWinners([url]);
expect(Number(winner.min_id)).toBe(pub.id);
expect(Number(winner.private_id)).toBe(Math.min(priv1.id, priv2.id));
});
it('returns [] for an empty or non-array candidate list', async () => {
expect(await appStore.listIndexUrlWinners([])).toEqual([]);
expect(await appStore.listIndexUrlWinners(null)).toEqual([]);
});
});
@@ -563,3 +563,90 @@ describe('SubdomainStore reads and writes', () => {
).resolves.toBeUndefined();
});
});
// -- getBySubdomains (batched) --
describe('SubdomainStore.getBySubdomains', () => {
it('is keyed by the requested names and omits names with no row', async () => {
const owner = await makeUser();
const a = `gbs-a-${Math.random().toString(36).slice(2, 8)}`;
const b = `gbs-b-${Math.random().toString(36).slice(2, 8)}`;
const missing = `gbs-missing-${Math.random().toString(36).slice(2, 8)}`;
await store.create({ userId: owner, subdomain: a } as never);
await store.create({ userId: owner, subdomain: b } as never);
const found = await store.getBySubdomains([a, b, missing]);
expect(found.size).toBe(2);
expect(found.get(a)?.subdomain).toBe(a);
expect(found.get(b)?.subdomain).toBe(b);
expect(found.has(missing)).toBe(false);
});
it('serves a cache hit with no DB read', async () => {
const owner = await makeUser();
const name = `gbs-cached-${Math.random().toString(36).slice(2, 8)}`;
// `create` writes through the cache, so the row is already warm.
await store.create({ userId: owner, subdomain: name } as never);
const read = vi.spyOn(server.clients.db, 'read');
const pread = vi.spyOn(server.clients.db, 'pread');
try {
const found = await store.getBySubdomains([name]);
expect(found.get(name)?.subdomain).toBe(name);
expect(read).not.toHaveBeenCalled();
expect(pread).not.toHaveBeenCalled();
} finally {
read.mockRestore();
pread.mockRestore();
}
});
it('a cached negative marker is absent from the result with no DB read', async () => {
const name = `gbs-neg-${Math.random().toString(36).slice(2, 8)}`;
// Poison the negative-cache marker first.
expect(await store.getBySubdomains([name])).toEqual(new Map());
const read = vi.spyOn(server.clients.db, 'read');
try {
const found = await store.getBySubdomains([name]);
expect(found.has(name)).toBe(false);
expect(read).not.toHaveBeenCalled();
} finally {
read.mockRestore();
}
});
it('treats a cached JSON null as a miss and reads the DB, like getBySubdomain', async () => {
const owner = await makeUser();
const name = `gbs-null-${Math.random().toString(36).slice(2, 8)}`;
await store.create({ userId: owner, subdomain: name } as never);
await server.clients.redis.set(cacheKey(name), 'null');
const found = await store.getBySubdomains([name]);
expect(found.get(name)?.subdomain).toBe(name);
});
it('a primary read uses pread and writes through, healing a stale negative marker', async () => {
const owner = await makeUser();
const name = `gbs-heal-${Math.random().toString(36).slice(2, 8)}`;
// Poison the cache with a negative marker, then create the row.
expect(await store.getBySubdomains([name])).toEqual(new Map());
await store.create({ userId: owner, subdomain: name } as never);
const pread = vi.spyOn(server.clients.db, 'pread');
try {
const primaryFound = await store.getBySubdomains([name], {
primary: true,
});
expect(primaryFound.get(name)?.subdomain).toBe(name);
expect(pread).toHaveBeenCalled();
} finally {
pread.mockRestore();
}
// The write-through heals the replica-path cache too.
const healedFound = await store.getBySubdomains([name]);
expect(healedFound.get(name)?.subdomain).toBe(name);
});
});
+160 -22
View File
@@ -111,6 +111,8 @@ const CACHE_TTL_SECONDS = 60 * 60;
// Sentinel so 404s on the same public subdomain don't hit the DB repeatedly.
const NEGATIVE_CACHE_MARKER = '__none__';
const NEGATIVE_CACHE_TTL_SECONDS = 10;
// Cap on placeholders per `IN (?, ?, …)` chunk — mirrors AppStore/UserStore.
const BULK_QUERY_CHUNK_SIZE = 200;
export class SubdomainStore extends PuterStore {
// -- Reads --------------------------------------------------------
@@ -170,34 +172,115 @@ export class SubdomainStore extends PuterStore {
: await this.clients.db.read(sql, [subdomain]);
const row = (rows[0] as unknown as SubdomainRow | undefined) ?? null;
// These writes are fire-and-forget, so a mutation that lands between
// the SELECT above and the SET below would otherwise be overwritten
// by the row we just read — stranding the pre-write row in cache for
// the full TTL (an hour of a site serving its old root_dir after
// `hosting.update`). A replica read is not authoritative, so it only
// *populates* an absent key (`NX`) and can never clobber a fresher
// write. A `primary` read is read-after-write on the primary, so it
// writes through — that's what heals a stale negative marker.
const populate = (value: string, ttlSeconds: number) =>
(primary
? this.clients.redis.set(cacheKey, value, 'EX', ttlSeconds)
: this.clients.redis.set(
cacheKey,
value,
'EX',
ttlSeconds,
'NX',
)
).catch(() => {});
if (row) {
populate(JSON.stringify(row), CACHE_TTL_SECONDS);
void this.#populateCache(
cacheKey,
JSON.stringify(row),
CACHE_TTL_SECONDS,
primary,
);
} else {
populate(NEGATIVE_CACHE_MARKER, NEGATIVE_CACHE_TTL_SECONDS);
void this.#populateCache(
cacheKey,
NEGATIVE_CACHE_MARKER,
NEGATIVE_CACHE_TTL_SECONDS,
primary,
);
}
return row;
}
/**
* Batched sibling of {@link getBySubdomain} — one cache pipeline plus one
* chunked `IN (…)` query for the misses, instead of one lookup per name.
* Keyed by the requested names; a name with no row (negative-cached or
* genuinely absent) is simply missing from the returned map.
*/
async getBySubdomains(
names: string[],
{ primary = false }: { primary?: boolean } = {},
): Promise<Map<string, SubdomainRow>> {
const result = new Map<string, SubdomainRow>();
const uniqueNames = [
...new Set(
(Array.isArray(names) ? names : []).filter(
(n): n is string => typeof n === 'string' && n.length > 0,
),
),
];
if (uniqueNames.length === 0) return result;
let missingNames = uniqueNames;
if (!primary) {
missingNames = [];
try {
const pipeline = this.clients.redis.pipeline();
for (const name of uniqueNames) {
pipeline.get(this.#cacheKey(name));
}
const cacheResults = (await pipeline.exec()) ?? [];
for (let i = 0; i < uniqueNames.length; i++) {
const name = uniqueNames[i]!;
const raw = cacheResults[i]?.[1];
if (raw === NEGATIVE_CACHE_MARKER) continue; // cached miss
if (typeof raw === 'string') {
try {
const parsed = JSON.parse(
raw,
) as SubdomainRow | null;
if (parsed) {
result.set(name, parsed);
continue;
}
} catch {
// Fall through to DB on any parse failure.
}
}
missingNames.push(name);
}
} catch {
missingNames = uniqueNames;
}
}
if (missingNames.length === 0) return result;
const rowsByName = new Map<string, SubdomainRow>();
for (
let offset = 0;
offset < missingNames.length;
offset += BULK_QUERY_CHUNK_SIZE
) {
const chunk = missingNames.slice(
offset,
offset + BULK_QUERY_CHUNK_SIZE,
);
const placeholders = chunk.map(() => '?').join(', ');
const sql = `SELECT * FROM \`subdomains\` WHERE \`subdomain\` IN (${placeholders})`;
const rows = (primary
? await this.clients.db.pread(sql, chunk)
: await this.clients.db.read(
sql,
chunk,
)) as unknown as SubdomainRow[];
// Keyed lowercase: a case-insensitive collation can return a row
// cased differently from the name, which `getBySubdomain` accepts.
for (const row of rows) {
if (typeof row.subdomain !== 'string') continue;
const key = row.subdomain.toLowerCase();
if (!rowsByName.has(key)) rowsByName.set(key, row);
}
}
for (const name of missingNames) {
const row = rowsByName.get(name.toLowerCase());
if (row) result.set(name, row);
}
void this.#populateCacheForNames(missingNames, result, primary);
return result;
}
async listByUserId(
userId: number,
{
@@ -634,6 +717,61 @@ export class SubdomainStore extends PuterStore {
return `${CACHE_KEY_PREFIX}:listByUserPrefixKeys:${userId}`;
}
/**
* Write-through on a primary read (heals a stale negative marker); `NX` on
* a replica read so an in-flight fresher write can't be clobbered by a
* stale value landing after it.
*/
async #populateCache(
cacheKey: string,
value: string,
ttlSeconds: number,
primary: boolean,
): Promise<void> {
try {
if (primary) {
await this.clients.redis.set(cacheKey, value, 'EX', ttlSeconds);
} else {
await this.clients.redis.set(
cacheKey,
value,
'EX',
ttlSeconds,
'NX',
);
}
} catch {
/* best-effort */
}
}
/** Pipelined sibling of {@link #populateCache} for `getBySubdomains`. */
async #populateCacheForNames(
names: string[],
rowsByName: Map<string, SubdomainRow>,
primary: boolean,
): Promise<void> {
try {
const pipeline = this.clients.redis.pipeline();
for (const name of names) {
const row = rowsByName.get(name);
const cacheKey = this.#cacheKey(name);
const value = row ? JSON.stringify(row) : NEGATIVE_CACHE_MARKER;
const ttl = row
? CACHE_TTL_SECONDS
: NEGATIVE_CACHE_TTL_SECONDS;
if (primary) {
pipeline.set(cacheKey, value, 'EX', ttl);
} else {
pipeline.set(cacheKey, value, 'EX', ttl, 'NX');
}
}
await pipeline.exec();
} catch {
/* best-effort */
}
}
async #refreshCache(row: { subdomain?: string }) {
if (!row?.subdomain) return;
await this.publishCacheKeys({
+161 -1
View File
@@ -17,13 +17,17 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { describe, expect, it, vi } from 'vitest';
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
import { v4 as uuidv4 } from 'uuid';
import { PuterServer } from '../server.js';
import { setupTestServer } from '../testUtil.js';
import {
buildHostedBackingDenial,
buildHostedSubdomainIndexUrlCandidates,
extractPuterHostedSubdomain,
getPuterHostedDomains,
hostedIndexUrlBackingIsUnavailable,
hostedIndexUrlBackingsAreUnavailable,
} from './hostedAppBacking.js';
// Pure-unit companion to the integration coverage in
@@ -354,3 +358,159 @@ describe('buildHostedBackingDenial', () => {
});
});
});
// Matches the hosting domains in config.default.json, so subdomain rows
// created through the real store resolve the same way `AppDriver` would see
// them in production.
const HOSTING_CONFIG = {
static_hosting_domain: 'site.puter.localhost',
static_hosting_domain_alt: 'host.puter.localhost',
private_app_hosting_domain: 'app.puter.localhost',
private_app_hosting_domain_alt: 'dev.puter.localhost',
};
const hostedUrl = (sub: string) => `https://${sub}.site.puter.localhost/`;
describe('hostedIndexUrlBackingsAreUnavailable (batched, against a real store)', () => {
let server: PuterServer;
beforeAll(async () => {
server = await setupTestServer();
});
afterAll(async () => {
await server?.shutdown();
});
const makeUser = async () => {
const username = `hab-${Math.random().toString(36).slice(2, 10)}`;
return server.stores.user.create({
username,
uuid: uuidv4(),
password: null,
email: `${username}@test.local`,
free_storage: 100 * 1024 * 1024,
requires_email_confirmation: false,
});
};
it('resolves a mix of apps in input order with one batched call', async () => {
const owner = await makeUser();
const attacker = await makeUser();
const liveSub = `live-${Math.random().toString(36).slice(2, 8)}`;
await server.stores.subdomain.create({
userId: owner.id,
subdomain: liveSub,
});
const reclaimedSub = `reclaimed-${Math.random().toString(36).slice(2, 8)}`;
await server.stores.subdomain.create({
userId: attacker.id,
subdomain: reclaimedSub,
});
const danglingSub = `dangling-${Math.random().toString(36).slice(2, 8)}`;
const external = {
index_url: 'https://elsewhere.example.com/',
owner_user_id: owner.id,
};
const live = { index_url: hostedUrl(liveSub), owner_user_id: owner.id };
const reclaimed = {
index_url: hostedUrl(reclaimedSub),
owner_user_id: owner.id,
};
const dangling = {
index_url: hostedUrl(danglingSub),
owner_user_id: owner.id,
};
const getBySubdomains = vi.spyOn(
server.stores.subdomain,
'getBySubdomains',
);
try {
const results = await hostedIndexUrlBackingsAreUnavailable({
apps: [external, live, reclaimed, dangling],
subdomainStore: server.stores.subdomain,
config: HOSTING_CONFIG,
});
// Input order preserved; external → false, live → false, a
// reclaimed owner → true, a gone subdomain → true.
expect(results).toEqual([false, false, true, true]);
// One non-primary batch carrying only the three hosted names —
// the external app never touches the store.
expect(getBySubdomains).toHaveBeenCalledTimes(2);
expect([...getBySubdomains.mock.calls[0]![0]].sort()).toEqual(
[liveSub, reclaimedSub, danglingSub].sort(),
);
// Primary follow-up only for the one name still missing.
expect(getBySubdomains.mock.calls[1]).toEqual([
[danglingSub],
{ primary: true },
]);
} finally {
getBySubdomains.mockRestore();
}
});
it('makes no store call when every app is non-hosted', async () => {
const owner = await makeUser();
const external = {
index_url: 'https://elsewhere.example.com/',
owner_user_id: owner.id,
};
const builtin = { index_url: undefined, owner_user_id: owner.id };
const getBySubdomains = vi.spyOn(
server.stores.subdomain,
'getBySubdomains',
);
try {
const results = await hostedIndexUrlBackingsAreUnavailable({
apps: [external, builtin],
subdomainStore: server.stores.subdomain,
config: HOSTING_CONFIG,
});
expect(results).toEqual([false, false]);
expect(getBySubdomains).not.toHaveBeenCalled();
} finally {
getBySubdomains.mockRestore();
}
});
it('does exactly one primary batch covering every still-missing name', async () => {
const owner = await makeUser();
const goneA = `gone-a-${Math.random().toString(36).slice(2, 6)}`;
const goneB = `gone-b-${Math.random().toString(36).slice(2, 6)}`;
const danglingA = {
index_url: hostedUrl(goneA),
owner_user_id: owner.id,
};
const danglingB = {
index_url: hostedUrl(goneB),
owner_user_id: owner.id,
};
const getBySubdomains = vi.spyOn(
server.stores.subdomain,
'getBySubdomains',
);
try {
const results = await hostedIndexUrlBackingsAreUnavailable({
apps: [danglingA, danglingB],
subdomainStore: server.stores.subdomain,
config: HOSTING_CONFIG,
});
expect(results).toEqual([true, true]);
expect(getBySubdomains).toHaveBeenCalledTimes(2);
expect(getBySubdomains.mock.calls[1]![1]).toEqual({
primary: true,
});
expect([...getBySubdomains.mock.calls[1]![0]].sort()).toEqual(
[goneA, goneB].sort(),
);
} finally {
getBySubdomains.mockRestore();
}
});
});
+56 -6
View File
@@ -181,6 +181,24 @@ export function extractPuterHostedSubdomain(
return null;
}
/**
* True when `row` (the subdomain backing `app`'s hosted index_url, or null)
* means the app's launch is unsafe: gone, or reclaimed by a different owner.
*/
function backingRowIsUnavailable(
app: AppBackingRow,
row: { user_id?: unknown } | null,
): boolean {
if (!row) return true; // subdomain no longer exists → dangling
const appOwnerId = Number(app.owner_user_id);
const subdomainOwnerId = Number(row.user_id);
if (!Number.isInteger(appOwnerId) || !Number.isInteger(subdomainOwnerId)) {
return true;
}
return subdomainOwnerId !== appOwnerId;
}
/**
* True when the app's puter-hosted subdomain is missing, or is currently owned
* by a different user than the app's owner (it was reclaimed — launching would
@@ -208,14 +226,46 @@ export async function hostedIndexUrlBackingIsUnavailable({
primary: true,
});
}
if (!row) return true; // subdomain no longer exists → dangling
return backingRowIsUnavailable(app, row);
}
const appOwnerId = Number(app.owner_user_id);
const subdomainOwnerId = Number(row.user_id);
if (!Number.isInteger(appOwnerId) || !Number.isInteger(subdomainOwnerId)) {
return true;
/**
* Batched sibling of {@link hostedIndexUrlBackingIsUnavailable} — one
* `getBySubdomains` call (plus a single primary follow-up for whatever's still
* missing) instead of per-app round trips. Returns a boolean per `apps` entry,
* in input order, with the same semantics as the single-app function above.
*/
export async function hostedIndexUrlBackingsAreUnavailable({
apps,
subdomainStore,
config,
}: {
apps: AppBackingRow[];
subdomainStore: Pick<SubdomainStore, 'getBySubdomains'>;
config: HostedDomainConfig | undefined | null;
}): Promise<boolean[]> {
const subdomains = apps.map((app) =>
extractPuterHostedSubdomain(app.index_url, config),
);
const names = [
...new Set(subdomains.filter((s): s is string => s !== null)),
];
if (names.length === 0) return apps.map(() => false);
let rowsByName = await subdomainStore.getBySubdomains(names);
const missing = names.filter((name) => !rowsByName.has(name));
if (missing.length > 0) {
const primaryRows = await subdomainStore.getBySubdomains(missing, {
primary: true,
});
rowsByName = new Map([...rowsByName, ...primaryRows]);
}
return subdomainOwnerId !== appOwnerId;
return apps.map((app, i) => {
const subdomain = subdomains[i];
if (!subdomain) return false;
return backingRowIsUnavailable(app, rowsByName.get(subdomain) ?? null);
});
}
/**