mirror of
https://github.com/HeyPuter/puter.git
synced 2026-10-10 22:01:40 +00:00
perf: batch per-app origin, canonical and hosted-backing lookups in app listings (PUT-1892) (#4013)
This commit is contained in:
1 parent
4f31388dad
commit
e6bda950f0
12 files changed
+1628
-123
No files matched your search
@@ -34,7 +34,7 @@ import {
|
||||
buildHostedBackingDenial,
|
||||
buildHostedSubdomainIndexUrlCandidates,
|
||||
extractPuterHostedSubdomain,
|
||||
hostedIndexUrlBackingIsUnavailable,
|
||||
hostedIndexUrlBackingsAreUnavailable,
|
||||
} from '../../util/hostedAppBacking.js';
|
||||
import {
|
||||
decodeCursor,
|
||||
@@ -119,6 +119,19 @@ const hasIndexUrlUniquenessExemption = (candidates) => {
|
||||
return false;
|
||||
};
|
||||
|
||||
/** Parsed `protocol//hostname[:port]` origin of an index_url, or null. */
|
||||
function indexUrlOrigin(indexUrl) {
|
||||
if (!indexUrl) return null;
|
||||
try {
|
||||
const parsed = new URL(indexUrl);
|
||||
return `${parsed.protocol}//${parsed.hostname}${
|
||||
parsed.port ? `:${parsed.port}` : ''
|
||||
}`;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Driver exposing the `puter-apps` interface.
|
||||
*
|
||||
@@ -407,20 +420,34 @@ export class AppDriver extends PuterDriver {
|
||||
// Pre-fetch in parallel:
|
||||
// - per-uid stats (already pipelined inside getAppsStats)
|
||||
// - filetype associations as a single IN-list query (was N queries)
|
||||
const [statsByUid, filetypesByAppId] = await Promise.all([
|
||||
// - canonical-index-url resolution and the hosted-backing check,
|
||||
// each batched across every visible app
|
||||
const [
|
||||
statsByUid,
|
||||
filetypesByAppId,
|
||||
canonicalByApp,
|
||||
hostedUnavailableByApp,
|
||||
] = await Promise.all([
|
||||
this.appStore.getAppsStats(visible.map((a) => a.uid)),
|
||||
this.appStore.getFiletypeAssociationsByIds(
|
||||
visible.map((a) => a.id),
|
||||
),
|
||||
this.#resolveCanonicalForIndexUrls(visible),
|
||||
this.#hostedBackingUnavailableFlags(visible),
|
||||
]);
|
||||
|
||||
const items = await Promise.all(
|
||||
visible.map((app) =>
|
||||
this.#toClient(app, actor, {
|
||||
...params,
|
||||
stats: statsByUid.get(app.uid),
|
||||
filetypes: filetypesByAppId.get(app.id) ?? [],
|
||||
}),
|
||||
visible.map((app, i) =>
|
||||
this.#toClient(
|
||||
app,
|
||||
actor,
|
||||
{ ...params, stats: statsByUid.get(app.uid) },
|
||||
{
|
||||
filetypes: filetypesByAppId.get(app.id) ?? [],
|
||||
canonical: canonicalByApp[i],
|
||||
hostedBackingUnavailable: hostedUnavailableByApp[i],
|
||||
},
|
||||
),
|
||||
),
|
||||
);
|
||||
if (!paginated) return items;
|
||||
@@ -891,9 +918,9 @@ export class AppDriver extends PuterDriver {
|
||||
// -- Serialization ------------------------------------------------
|
||||
|
||||
/**
|
||||
* Resolve the canonical app row that backs `app.index_url`.
|
||||
*
|
||||
* Returns `{ origin, expectedUid, canonicalApp }`:
|
||||
* Resolve the canonical app row backing each app's `index_url`, batched
|
||||
* across `apps`. Returns an array aligned with `apps`, each entry `{
|
||||
* origin, expectedUid, canonicalApp }`:
|
||||
*
|
||||
* - `origin` — the parsed origin string from `index_url`.
|
||||
* - `expectedUid` — the canonical app uid for that origin (oldest
|
||||
@@ -913,61 +940,94 @@ export class AppDriver extends PuterDriver {
|
||||
* pre-existing data from before the `subdomain_not_owned` check leaks
|
||||
* the victim's index_url.
|
||||
*
|
||||
* Returns `null` when there's no `index_url` or it doesn't parse.
|
||||
* An entry is `null` when its app has no `index_url` or it doesn't parse.
|
||||
*/
|
||||
async #resolveCanonicalForIndexUrl(app) {
|
||||
if (!app.index_url) return null;
|
||||
let origin;
|
||||
try {
|
||||
const parsed = new URL(app.index_url);
|
||||
origin = `${parsed.protocol}//${parsed.hostname}${
|
||||
parsed.port ? `:${parsed.port}` : ''
|
||||
}`;
|
||||
} catch {
|
||||
return null;
|
||||
async #resolveCanonicalForIndexUrls(apps) {
|
||||
const origins = apps.map((app) => indexUrlOrigin(app.index_url));
|
||||
const uniqueOrigins = [...new Set(origins.filter((o) => o !== null))];
|
||||
|
||||
let uidByOrigin = new Map();
|
||||
if (uniqueOrigins.length > 0) {
|
||||
try {
|
||||
uidByOrigin =
|
||||
await this.services.auth.appUidsFromOrigins(uniqueOrigins);
|
||||
} catch {
|
||||
uidByOrigin = new Map();
|
||||
}
|
||||
}
|
||||
try {
|
||||
const expectedUid =
|
||||
await this.services.auth.appUidFromOrigin(origin);
|
||||
// Avoid a needless DB hit on the self-match common case —
|
||||
// `app` is already the row we'd be re-fetching.
|
||||
const canonicalApp =
|
||||
expectedUid && expectedUid !== app.uid
|
||||
? await this.appStore.getByUid(expectedUid)
|
||||
: app;
|
||||
return { origin, expectedUid, canonicalApp };
|
||||
} catch {
|
||||
return null;
|
||||
|
||||
// Avoid a needless DB hit on the self-match common case — `app` is
|
||||
// already the row we'd be re-fetching.
|
||||
const uidsToFetch = new Set();
|
||||
for (let i = 0; i < apps.length; i++) {
|
||||
if (!origins[i]) continue;
|
||||
const expectedUid = uidByOrigin.get(origins[i]) ?? null;
|
||||
if (expectedUid && expectedUid !== apps[i].uid) {
|
||||
uidsToFetch.add(expectedUid);
|
||||
}
|
||||
}
|
||||
|
||||
let canonicalAppByUid = new Map();
|
||||
if (uidsToFetch.size > 0) {
|
||||
try {
|
||||
canonicalAppByUid = await this.appStore.getByUids([
|
||||
...uidsToFetch,
|
||||
]);
|
||||
} catch {
|
||||
canonicalAppByUid = new Map();
|
||||
}
|
||||
}
|
||||
|
||||
return apps.map((app, i) => {
|
||||
const origin = origins[i];
|
||||
if (!origin) return null;
|
||||
const expectedUid = uidByOrigin.get(origin) ?? null;
|
||||
if (!expectedUid) return null;
|
||||
if (expectedUid === app.uid) {
|
||||
return { origin, expectedUid, canonicalApp: app };
|
||||
}
|
||||
return {
|
||||
origin,
|
||||
expectedUid,
|
||||
canonicalApp: canonicalAppByUid.get(expectedUid) ?? null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Launch-safety check for puter-hosted `index_url`s. See
|
||||
* `util/hostedAppBacking.ts` — the check lives there because every producer
|
||||
* of launchable app metadata needs it, not just this driver.
|
||||
* Launch-safety check for puter-hosted `index_url`s, batched across `apps`.
|
||||
* See `util/hostedAppBacking.ts` — the check lives there because every
|
||||
* producer of launchable app metadata needs it, not just this driver.
|
||||
*/
|
||||
async #hostedIndexUrlBackingIsUnavailable(app) {
|
||||
return hostedIndexUrlBackingIsUnavailable({
|
||||
app,
|
||||
async #hostedBackingUnavailableFlags(apps) {
|
||||
return hostedIndexUrlBackingsAreUnavailable({
|
||||
apps,
|
||||
subdomainStore: this.stores.subdomain,
|
||||
config: this.config,
|
||||
});
|
||||
}
|
||||
|
||||
async #toClient(app, actor, params = {}) {
|
||||
async #toClient(app, actor, params = {}, prefetched = {}) {
|
||||
if (!app) return null;
|
||||
|
||||
// `select` pre-fetches filetypes for every visible app in one
|
||||
// batched query and threads them through `params.filetypes` to
|
||||
// avoid the N+1 in this hot loop. Single-app callers (`read`,
|
||||
// `create`, `update`) fall back to the per-app query.
|
||||
// `select` batches these lookups per page and passes them in
|
||||
// `prefetched`; single-app callers resolve them here. Never read them
|
||||
// from `params`, which carries the RPC caller's input.
|
||||
const [filetypes, canonicalForIndexUrl, hostedBackingUnavailable] =
|
||||
await Promise.all([
|
||||
params.filetypes !== undefined
|
||||
? Promise.resolve(params.filetypes)
|
||||
prefetched.filetypes !== undefined
|
||||
? Promise.resolve(prefetched.filetypes)
|
||||
: this.appStore.getFiletypeAssociations(app.id),
|
||||
this.#resolveCanonicalForIndexUrl(app),
|
||||
this.#hostedIndexUrlBackingIsUnavailable(app),
|
||||
prefetched.canonical !== undefined
|
||||
? Promise.resolve(prefetched.canonical)
|
||||
: this.#resolveCanonicalForIndexUrls([app]).then(
|
||||
(r) => r[0],
|
||||
),
|
||||
prefetched.hostedBackingUnavailable !== undefined
|
||||
? Promise.resolve(prefetched.hostedBackingUnavailable)
|
||||
: this.#hostedBackingUnavailableFlags([app]).then(
|
||||
(r) => r[0],
|
||||
),
|
||||
]);
|
||||
|
||||
const createdFromOrigin =
|
||||
|
||||
@@ -1983,3 +1983,358 @@ describe('AppDriver hosted-subdomain ownership check', () => {
|
||||
).not.toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// -- select: batched #toClient query counts --
|
||||
//
|
||||
// Canonical index_url resolution and the hosted-backing check are batched per
|
||||
// page, so these counts must stay flat as the page grows.
|
||||
|
||||
describe('AppDriver.select query-count regression', () => {
|
||||
const makeMixedApps = async (count: number) => {
|
||||
const { actor, userId } = await makeUser();
|
||||
for (let i = 0; i < count; i++) {
|
||||
const kind = i % 3;
|
||||
if (kind === 0) {
|
||||
// Live hosted: owns the subdomain it points at.
|
||||
const sub = uniqueName(`qclive${i}`);
|
||||
await server.stores.subdomain.create({ userId, subdomain: sub });
|
||||
await withActor(actor, () =>
|
||||
driver.create({
|
||||
object: {
|
||||
name: uniqueName(`qc-live-${i}`),
|
||||
title: 't',
|
||||
index_url: `https://${sub}.site.puter.localhost/`,
|
||||
},
|
||||
}),
|
||||
);
|
||||
} else if (kind === 1) {
|
||||
// Dangling hosted: the subdomain is gone by the time we read.
|
||||
const sub = uniqueName(`qcdang${i}`);
|
||||
const row = await server.stores.subdomain.create({
|
||||
userId,
|
||||
subdomain: sub,
|
||||
});
|
||||
await withActor(actor, () =>
|
||||
driver.create({
|
||||
object: {
|
||||
name: uniqueName(`qc-dangling-${i}`),
|
||||
title: 't',
|
||||
index_url: `https://${sub}.site.puter.localhost/`,
|
||||
},
|
||||
}),
|
||||
);
|
||||
await server.stores.subdomain.deleteByUuid(
|
||||
String((row as { uuid: string }).uuid),
|
||||
{ userId },
|
||||
);
|
||||
} else {
|
||||
// External, with a path — not puter-hosted at all.
|
||||
await withActor(actor, () =>
|
||||
driver.create({
|
||||
object: {
|
||||
name: uniqueName(`qc-ext-${i}`),
|
||||
title: 't',
|
||||
index_url: `${uniqueIndexUrl()}some/path`,
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
return actor;
|
||||
};
|
||||
|
||||
const countQueriesForSelect = async (actor: Actor) => {
|
||||
const read = vi.spyOn(server.clients.db, 'read');
|
||||
const pread = vi.spyOn(server.clients.db, 'pread');
|
||||
try {
|
||||
const items = (await withActor(actor, () =>
|
||||
driver.select({ predicate: ['user-can-edit'] }),
|
||||
)) as Array<Record<string, unknown>>;
|
||||
const counts = { indexUrlIn: 0, subdomains: 0, appsUidEq: 0 };
|
||||
for (const call of read.mock.calls) {
|
||||
const sql = call[0] as string;
|
||||
if (/`index_url`\s+IN\s*\(/i.test(sql)) {
|
||||
counts.indexUrlIn++;
|
||||
} else if (sql.includes('`subdomains`')) {
|
||||
counts.subdomains++;
|
||||
} else if (
|
||||
sql.includes('`apps`') &&
|
||||
/`uid`\s*=\s*\?/.test(sql)
|
||||
) {
|
||||
counts.appsUidEq++;
|
||||
}
|
||||
}
|
||||
return {
|
||||
counts,
|
||||
pread: pread.mock.calls.length,
|
||||
itemCount: items.length,
|
||||
};
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
pread.mockRestore();
|
||||
}
|
||||
};
|
||||
|
||||
it('keeps index_url / subdomains / per-app-uid query counts constant from 3 apps to 30 apps', async () => {
|
||||
const actor3 = await makeMixedApps(3);
|
||||
const actor30 = await makeMixedApps(30);
|
||||
|
||||
const small = await countQueriesForSelect(actor3);
|
||||
const large = await countQueriesForSelect(actor30);
|
||||
|
||||
expect(small.itemCount).toBe(3);
|
||||
expect(large.itemCount).toBe(30);
|
||||
// Sanity check the classifier actually saw the shapes it's counting.
|
||||
expect(small.counts.indexUrlIn).toBeGreaterThan(0);
|
||||
expect(small.counts.subdomains).toBeGreaterThan(0);
|
||||
|
||||
expect(large.counts).toEqual(small.counts);
|
||||
expect(large.pread).toBe(small.pread);
|
||||
});
|
||||
});
|
||||
|
||||
// -- select / read parity --
|
||||
|
||||
describe('AppDriver.select / read parity', () => {
|
||||
const findViaBroadSelect = async (
|
||||
actor: Actor,
|
||||
uid: string,
|
||||
): Promise<Record<string, unknown> | undefined> => {
|
||||
let cursor: string | null | undefined = null;
|
||||
do {
|
||||
const page = (await withActor(actor, () =>
|
||||
driver.select({ limit: 50, cursor }),
|
||||
)) as { items: Array<Record<string, unknown>>; cursor?: string };
|
||||
const found = page.items.find((r) => r.uid === uid);
|
||||
if (found) return found;
|
||||
cursor = page.cursor;
|
||||
} while (cursor);
|
||||
return undefined;
|
||||
};
|
||||
|
||||
it('every select item (minus stats) deep-equals the corresponding read', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const names: string[] = [];
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const name = uniqueName(`parity${i}`);
|
||||
names.push(name);
|
||||
await withActor(actor, () =>
|
||||
driver.create({
|
||||
object: { name, title: 't', index_url: uniqueIndexUrl() },
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const selectResult = (await withActor(actor, () =>
|
||||
driver.select({ predicate: ['user-can-edit'] }),
|
||||
)) as Array<Record<string, unknown>>;
|
||||
const ours = selectResult.filter((item) =>
|
||||
names.includes(item.name as string),
|
||||
);
|
||||
expect(ours.length).toBe(names.length);
|
||||
|
||||
for (const item of ours) {
|
||||
const read = await withActor(actor, () =>
|
||||
driver.read({ uid: item.uid as string }),
|
||||
);
|
||||
const { stats: _itemStats, ...itemRest } = item;
|
||||
const { stats: _readStats, ...readRest } = read;
|
||||
expect(itemRest).toEqual(readRest);
|
||||
}
|
||||
});
|
||||
|
||||
it('sets created_from_origin on the canonical hosted row and null on a duplicate', async () => {
|
||||
const { actor, userId } = await makeUser();
|
||||
const sub = uniqueName('cfo');
|
||||
await server.stores.subdomain.create({ userId, subdomain: sub });
|
||||
const url = `https://${sub}.site.puter.localhost/`;
|
||||
const canonical = await withActor(actor, () =>
|
||||
driver.create({
|
||||
object: {
|
||||
name: uniqueName('cfo-canon'),
|
||||
title: 't',
|
||||
index_url: url,
|
||||
},
|
||||
}),
|
||||
);
|
||||
// A duplicate row at the same index_url — `create` would normally
|
||||
// refuse this; direct insert mirrors the pre-existing-data shape
|
||||
// the canonical resolver has to cope with.
|
||||
const dupUid = `app-${uuidv4()}`;
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`) VALUES (?, ?, ?, ?, ?)',
|
||||
[dupUid, uniqueName('cfo-dup'), 'dup', url, userId],
|
||||
);
|
||||
|
||||
const result = (await withActor(actor, () =>
|
||||
driver.select({ predicate: ['user-can-edit'] }),
|
||||
)) as Array<Record<string, unknown>>;
|
||||
const canonItem = result.find((r) => r.uid === canonical.uid);
|
||||
const dupItem = result.find((r) => r.uid === dupUid);
|
||||
expect(canonItem?.created_from_origin).toBe(
|
||||
`https://${sub}.site.puter.localhost`,
|
||||
);
|
||||
expect(dupItem?.created_from_origin).toBeNull();
|
||||
});
|
||||
|
||||
it('gates the canonical-private row: a public duplicate withholds index_url and denies access', async () => {
|
||||
const ownerA = await makeUser();
|
||||
const ownerB = await makeUser();
|
||||
|
||||
const sharedUrl = uniqueIndexUrl();
|
||||
const uidA = `app-${uuidv4()}`;
|
||||
const uidB = `app-${uuidv4()}`;
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`, `is_private`) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
[
|
||||
uidA,
|
||||
uniqueName('priv-a'),
|
||||
'Private A',
|
||||
sharedUrl,
|
||||
ownerA.userId,
|
||||
1,
|
||||
],
|
||||
);
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`, `is_private`) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
[
|
||||
uidB,
|
||||
uniqueName('pub-b'),
|
||||
'Public B',
|
||||
sharedUrl,
|
||||
ownerB.userId,
|
||||
0,
|
||||
],
|
||||
);
|
||||
|
||||
// B's own owner is not A's owner, so the gate still applies to them.
|
||||
const result = (await withActor(ownerB.actor, () =>
|
||||
driver.select({ predicate: ['user-can-edit'] }),
|
||||
)) as Array<Record<string, unknown>>;
|
||||
const bItem = result.find((r) => r.uid === uidB);
|
||||
expect(bItem).toBeTruthy();
|
||||
expect(bItem!.index_url).toBeUndefined();
|
||||
expect(
|
||||
(bItem!.privateAccess as { hasAccess?: boolean }).hasAccess,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('dangling hosted app via select: owner keeps index_url, others do not', async () => {
|
||||
const owner = await makeUser();
|
||||
const other = await makeUser();
|
||||
const sub = uniqueName('qcdangsel');
|
||||
const row = await server.stores.subdomain.create({
|
||||
userId: owner.userId,
|
||||
subdomain: sub,
|
||||
});
|
||||
const created = await withActor(owner.actor, () =>
|
||||
driver.create({
|
||||
object: {
|
||||
name: uniqueName('dangling-sel'),
|
||||
title: 't',
|
||||
index_url: `https://${sub}.site.puter.localhost/`,
|
||||
},
|
||||
}),
|
||||
);
|
||||
await server.stores.subdomain.deleteByUuid(
|
||||
String((row as { uuid: string }).uuid),
|
||||
{ userId: owner.userId },
|
||||
);
|
||||
|
||||
const ownerResult = (await withActor(owner.actor, () =>
|
||||
driver.select({ predicate: ['user-can-edit'] }),
|
||||
)) as Array<Record<string, unknown>>;
|
||||
const ownerItem = ownerResult.find((r) => r.uid === created.uid);
|
||||
expect(ownerItem?.index_url).toBe(created.index_url);
|
||||
expect(
|
||||
(ownerItem?.privateAccess as { reason?: string } | undefined)
|
||||
?.reason,
|
||||
).toBe('hosted_backing_unavailable');
|
||||
|
||||
const otherItem = await findViaBroadSelect(
|
||||
other.actor,
|
||||
created.uid as string,
|
||||
);
|
||||
expect(otherItem?.index_url).toBeUndefined();
|
||||
expect(
|
||||
(otherItem?.privateAccess as { hasAccess?: boolean } | undefined)
|
||||
?.hasAccess,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('a blocked origin resolves created_from_origin to null without select throwing', async () => {
|
||||
const { actor } = await makeUser();
|
||||
const blockedHost = `blocked-${Math.random().toString(36).slice(2, 10)}.test`;
|
||||
const created = await withActor(actor, () =>
|
||||
driver.create({
|
||||
object: {
|
||||
name: uniqueName('blocked-sel'),
|
||||
title: 't',
|
||||
index_url: `https://${blockedHost}/app`,
|
||||
},
|
||||
}),
|
||||
);
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `blocked_app_origins` (`domain`, `include_subdomains`) VALUES (?, ?)',
|
||||
[blockedHost, 0],
|
||||
);
|
||||
(
|
||||
server.services.appOriginBlocklist as { invalidate: () => void }
|
||||
).invalidate();
|
||||
|
||||
const result = (await withActor(actor, () =>
|
||||
driver.select({ predicate: ['user-can-edit'] }),
|
||||
)) as Array<Record<string, unknown>>;
|
||||
const item = result.find((r) => r.uid === created.uid);
|
||||
expect(item).toBeTruthy();
|
||||
expect(item!.created_from_origin).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// -- read: prefetched-shaped params cannot be spoofed --
|
||||
|
||||
describe('AppDriver.read prefetched-param isolation', () => {
|
||||
it('ignores caller-supplied hostedBackingUnavailable/canonical/filetypes on a dangling app', async () => {
|
||||
const owner = await makeUser();
|
||||
const attacker = await makeUser();
|
||||
const sub = uniqueName('sec-dangling');
|
||||
const row = await server.stores.subdomain.create({
|
||||
userId: owner.userId,
|
||||
subdomain: sub,
|
||||
});
|
||||
const created = await withActor(owner.actor, () =>
|
||||
driver.create({
|
||||
object: {
|
||||
name: uniqueName('sec-app'),
|
||||
title: 't',
|
||||
index_url: `https://${sub}.site.puter.localhost/`,
|
||||
filetype_associations: ['.puter'],
|
||||
},
|
||||
}),
|
||||
);
|
||||
await server.stores.subdomain.deleteByUuid(
|
||||
String((row as { uuid: string }).uuid),
|
||||
{ userId: owner.userId },
|
||||
);
|
||||
|
||||
const result = await withActor(attacker.actor, () =>
|
||||
driver.read({
|
||||
uid: created.uid,
|
||||
params: {
|
||||
hostedBackingUnavailable: false,
|
||||
canonical: null,
|
||||
filetypes: ['x'],
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
// The spoofed `hostedBackingUnavailable: false` doesn't suppress the
|
||||
// real denial, so the dangling app's index_url is still withheld.
|
||||
expect(result.index_url).toBeUndefined();
|
||||
// The spoofed `filetypes` doesn't override the DB-backed list.
|
||||
expect(result.filetype_associations).toEqual(
|
||||
expect.arrayContaining(['puter']),
|
||||
);
|
||||
expect(result.filetype_associations).not.toContain('x');
|
||||
});
|
||||
});
|
||||
@@ -1074,3 +1074,48 @@ describe('SubdomainDriver associated_app derivation', () => {
|
||||
expect(read.associated_app).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// -- associated_app derivation at scale --
|
||||
//
|
||||
// ~24 index_url candidates per subdomain: a page this size must be chunked
|
||||
// to stay under SQLite's bound-parameter limit.
|
||||
|
||||
describe('SubdomainDriver.select at scale', () => {
|
||||
it('resolves associated_app for 1,400 subdomains without tripping the SQL variable limit', async () => {
|
||||
const { actor, userId } = await makeUser();
|
||||
const prefix = `scale-${Math.random().toString(36).slice(2, 8)}-`;
|
||||
const count = 1400;
|
||||
const names: string[] = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
names.push(`${prefix}${i}`);
|
||||
}
|
||||
|
||||
// Bulk-insert directly — bypasses the driver's per-user quota, the
|
||||
// same way a bulk-provisioned fleet of sites would exist in prod.
|
||||
const placeholders = names.map(() => '(?, ?, ?)').join(', ');
|
||||
const values: unknown[] = [];
|
||||
for (const name of names) {
|
||||
values.push(uuidv4(), name, userId);
|
||||
}
|
||||
await server.clients.db.write(
|
||||
`INSERT INTO \`subdomains\` (\`uuid\`, \`subdomain\`, \`user_id\`) VALUES ${placeholders}`,
|
||||
values,
|
||||
);
|
||||
|
||||
const lastSub = names[names.length - 1]!;
|
||||
const app = await createAppWithIndexUrl(
|
||||
userId,
|
||||
`http://${lastSub}.site.puter.localhost/`,
|
||||
);
|
||||
|
||||
const result = (await withActor(actor, () =>
|
||||
driver.select({}),
|
||||
)) as Array<Record<string, unknown>>;
|
||||
|
||||
expect(result.length).toBe(count);
|
||||
const lastItem = result.find((r) => r.subdomain === lastSub);
|
||||
expect(
|
||||
(lastItem?.associated_app as { uid: string } | null)?.uid,
|
||||
).toBe(app.uid);
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -188,10 +188,9 @@ export class SubdomainDriver extends PuterDriver {
|
||||
// they have open is not them.
|
||||
// See `util/hostedAppBacking.ts` for the wider rule.
|
||||
//
|
||||
// Last check before the insert on purpose: `apps.index_url` is
|
||||
// unindexed, so this scan only runs for a request that would otherwise
|
||||
// have created the row, and it stays behind the same root_dir gate as
|
||||
// the existing uniqueness answer.
|
||||
// Last check before the insert on purpose: it stays behind the same
|
||||
// root_dir gate as the existing uniqueness answer, so it only runs
|
||||
// for a request that would otherwise have created the row.
|
||||
const appsHoldingName = (await this.stores.app.listByIndexUrlCandidates(
|
||||
buildHostedSubdomainIndexUrlCandidates(subdomain, this.config),
|
||||
)) as Array<Record<string, unknown>>;
|
||||
@@ -737,7 +736,8 @@ export class SubdomainDriver extends PuterDriver {
|
||||
* hosting domains × protocols × paths). Returns a `rowUuid → appId` map.
|
||||
* Rows with no matching app are absent.
|
||||
*
|
||||
* Runs one batched DB query regardless of input size.
|
||||
* Goes through the batched `listByIndexUrlCandidates` store lookup rather
|
||||
* than one query per row, regardless of input size.
|
||||
*/
|
||||
async #deriveAssociatedAppIdByRowUuid(
|
||||
rows: Array<Record<string, unknown>>,
|
||||
@@ -782,16 +782,14 @@ export class SubdomainDriver extends PuterDriver {
|
||||
return result;
|
||||
}
|
||||
|
||||
const userIds = [...userIdToRowMeta.keys()];
|
||||
const userPlaceholders = userIds.map(() => '?').join(', ');
|
||||
const candidateList = [...allCandidates];
|
||||
const urlPlaceholders = candidateList.map(() => '?').join(', ');
|
||||
const matches = (await this.clients.db.read(
|
||||
`SELECT \`id\`, \`owner_user_id\`, \`index_url\` FROM \`apps\`
|
||||
WHERE \`owner_user_id\` IN (${userPlaceholders})
|
||||
AND \`index_url\` IN (${urlPlaceholders})`,
|
||||
[...userIds, ...candidateList],
|
||||
const matches = (await this.stores.app.listByIndexUrlCandidates(
|
||||
candidateList,
|
||||
)) as Array<Record<string, unknown>>;
|
||||
// Oldest row wins when more than one app matches the same candidate
|
||||
// for the same owner — the owner-match filter below is what actually
|
||||
// restricts matches to rows this batch cares about.
|
||||
matches.sort((a, b) => Number(a.id) - Number(b.id));
|
||||
|
||||
for (const m of matches) {
|
||||
const appId = typeof m.id === 'number' ? m.id : Number(m.id);
|
||||
|
||||
@@ -1811,6 +1811,155 @@ describe('AuthService (integration)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('appUidsFromOrigins', () => {
|
||||
it('matches appUidFromOrigin across hosting variants, an unknown external, and a repointed origin', async () => {
|
||||
const sub = `batch-${Math.random().toString(36).slice(2, 10)}`;
|
||||
const origins = [
|
||||
`https://${sub}.site.puter.localhost`,
|
||||
`https://${sub}.host.puter.localhost`,
|
||||
`http://${sub}.app.puter.localhost`,
|
||||
`https://external-${uuidv4()}.example.com`,
|
||||
`chrome-extension://${uuidv4()}`,
|
||||
];
|
||||
|
||||
const repointOrigin = `https://repoint-batch-${uuidv4()}.example.com`;
|
||||
const repointed = await server.stores.app.createFromOrigin(
|
||||
await authService.appUidFromOrigin(repointOrigin),
|
||||
authService.canonicalizeOrigin(repointOrigin),
|
||||
);
|
||||
await server.stores.app.update(repointed.id, {
|
||||
index_url: `https://repoint-new-${uuidv4()}.example.com`,
|
||||
});
|
||||
origins.push(repointOrigin);
|
||||
|
||||
const expected = new Map<string, string>();
|
||||
for (const origin of origins) {
|
||||
expected.set(
|
||||
origin,
|
||||
await authService.appUidFromOrigin(origin),
|
||||
);
|
||||
}
|
||||
|
||||
const batched = await authService.appUidsFromOrigins(origins);
|
||||
for (const origin of origins) {
|
||||
expect(batched.get(origin)).toBe(expected.get(origin));
|
||||
}
|
||||
// Every hosting variant of the same subdomain collapses to one uid.
|
||||
expect(batched.get(origins[0])).toBe(batched.get(origins[1]));
|
||||
expect(batched.get(origins[0])).toBe(batched.get(origins[2]));
|
||||
// The repointed origin moved on to a successor uid, not the
|
||||
// stale row's.
|
||||
expect(batched.get(repointOrigin)).not.toBe(repointed.uid);
|
||||
});
|
||||
|
||||
it('resolves unparseable origins to null instead of throwing', async () => {
|
||||
const result = await authService.appUidsFromOrigins([
|
||||
'not-a-url',
|
||||
'javascript:alert(document.domain)',
|
||||
]);
|
||||
expect(result.get('not-a-url')).toBeNull();
|
||||
expect(result.get('javascript:alert(document.domain)')).toBeNull();
|
||||
});
|
||||
|
||||
it('resolves a blocked origin to null without throwing or affecting other origins', async () => {
|
||||
const blockedHost = `blocked-batch-${uuidv4()}.example.com`;
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `blocked_app_origins` (`domain`, `include_subdomains`) VALUES (?, ?)',
|
||||
[blockedHost, 0],
|
||||
);
|
||||
(
|
||||
server.services.appOriginBlocklist as { invalidate: () => void }
|
||||
).invalidate();
|
||||
|
||||
const okOrigin = `https://ok-batch-${uuidv4()}.example.com`;
|
||||
const expectedOk = await authService.appUidFromOrigin(okOrigin);
|
||||
|
||||
const result = await authService.appUidsFromOrigins([
|
||||
`https://${blockedHost}`,
|
||||
okOrigin,
|
||||
]);
|
||||
expect(result.get(`https://${blockedHost}`)).toBeNull();
|
||||
expect(result.get(okOrigin)).toBe(expectedOk);
|
||||
});
|
||||
|
||||
it('prefers a private row over an older public stub, breaking ties by lowest id, in one batch', async () => {
|
||||
const user = await makeUser();
|
||||
const sub = `batch-priv-${Math.random().toString(36).slice(2, 10)}`;
|
||||
await server.stores.app.createFromOrigin(
|
||||
`app-${uuidv4()}`,
|
||||
`https://${sub}.host.puter.localhost`,
|
||||
{ ownerUserId: user.id },
|
||||
);
|
||||
const realUid = `app-${uuidv4()}`;
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `owner_user_id`, `is_private`) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
[
|
||||
realUid,
|
||||
`real-${sub}`,
|
||||
'Real app',
|
||||
`https://${sub}.app.puter.localhost`,
|
||||
user.id,
|
||||
1,
|
||||
],
|
||||
);
|
||||
|
||||
const origin = `https://${sub}.host.puter.localhost`;
|
||||
const result = await authService.appUidsFromOrigins([origin]);
|
||||
expect(result.get(origin)).toBe(realUid);
|
||||
});
|
||||
|
||||
it('a private row on one index_url variant wins over a lower-id public row on another, in both paths', async () => {
|
||||
// Both rows match the same origin's candidate set but live under
|
||||
// different exact `index_url` strings — the winners reduction has
|
||||
// to look across every matching variant, not just one.
|
||||
const base = `https://variant-${uuidv4()}.example.com`;
|
||||
const publicUid = `app-${uuidv4()}`;
|
||||
const privateUid = `app-${uuidv4()}`;
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `is_private`) VALUES (?, ?, ?, ?, ?)',
|
||||
[publicUid, `pub-${uuidv4()}`, 'Public', `${base}/`, 0],
|
||||
);
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `is_private`) VALUES (?, ?, ?, ?, ?)',
|
||||
[
|
||||
privateUid,
|
||||
`priv-${uuidv4()}`,
|
||||
'Private',
|
||||
`${base}/index.html`,
|
||||
1,
|
||||
],
|
||||
);
|
||||
|
||||
await expect(authService.appUidFromOrigin(base)).resolves.toBe(
|
||||
privateUid,
|
||||
);
|
||||
const batched = await authService.appUidsFromOrigins([base]);
|
||||
expect(batched.get(base)).toBe(privateUid);
|
||||
});
|
||||
|
||||
it('does not cross-assign uids between origins that each have their own canonical row', async () => {
|
||||
const subA = `batch-cross-a-${Math.random().toString(36).slice(2, 8)}`;
|
||||
const subB = `batch-cross-b-${Math.random().toString(36).slice(2, 8)}`;
|
||||
const originA = `https://${subA}.site.puter.localhost`;
|
||||
const originB = `https://${subB}.site.puter.localhost`;
|
||||
const appA = await server.stores.app.createFromOrigin(
|
||||
await authService.appUidFromOrigin(originA),
|
||||
authService.canonicalizeOrigin(originA),
|
||||
);
|
||||
const appB = await server.stores.app.createFromOrigin(
|
||||
await authService.appUidFromOrigin(originB),
|
||||
authService.canonicalizeOrigin(originB),
|
||||
);
|
||||
|
||||
const result = await authService.appUidsFromOrigins([
|
||||
originA,
|
||||
originB,
|
||||
]);
|
||||
expect(result.get(originA)).toBe(appA.uid);
|
||||
expect(result.get(originB)).toBe(appB.uid);
|
||||
});
|
||||
});
|
||||
|
||||
describe('subdomainOwnerIdFromOrigin', () => {
|
||||
// Test servers inherit the four hosting domains (production:
|
||||
// puter.site / puter.host / puter.app / puter.dev) from
|
||||
@@ -2937,6 +3086,23 @@ describe('AuthService.appUidFromOrigin — aliased hosts', () => {
|
||||
);
|
||||
expect(withPort).not.toBe(withoutPort);
|
||||
});
|
||||
|
||||
it('appUidsFromOrigins collapses an alias group onto one uid, batched', async () => {
|
||||
const alphaOrigin = 'https://alpha.example.com';
|
||||
const betaOrigin = 'https://beta.example.com';
|
||||
const ungroupedOrigin = 'https://gamma.example.com';
|
||||
|
||||
const result = await authService.appUidsFromOrigins([
|
||||
alphaOrigin,
|
||||
betaOrigin,
|
||||
ungroupedOrigin,
|
||||
]);
|
||||
expect(result.get(betaOrigin)).toBe(result.get(alphaOrigin));
|
||||
expect(result.get(ungroupedOrigin)).not.toBe(result.get(alphaOrigin));
|
||||
expect(result.get(alphaOrigin)).toBe(
|
||||
await authService.appUidFromOrigin(alphaOrigin),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('AuthService.subdomainOwnerIdFromOrigin — edge cases', () => {
|
||||
|
||||
@@ -819,6 +819,102 @@ export class AuthService extends PuterService {
|
||||
return this.#derivedAppUidForOrigin(appOrigin);
|
||||
}
|
||||
|
||||
/**
|
||||
* Batched sibling of {@link appUidFromOrigin} for listing paths that need a
|
||||
* uid per origin without a round trip each. Unparseable or blocked origins
|
||||
* resolve to null instead of throwing.
|
||||
*/
|
||||
async appUidsFromOrigins(
|
||||
origins: string[],
|
||||
): Promise<Map<string, string | null>> {
|
||||
const result = new Map<string, string | null>();
|
||||
const uniqueOrigins = [...new Set(origins)];
|
||||
|
||||
const appOriginByOrigin = new Map<string, string | null>();
|
||||
await Promise.all(
|
||||
uniqueOrigins.map(async (origin) => {
|
||||
try {
|
||||
appOriginByOrigin.set(
|
||||
origin,
|
||||
await this.#appOriginFor(origin),
|
||||
);
|
||||
} catch {
|
||||
appOriginByOrigin.set(origin, null);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
const uniqueAppOrigins = [
|
||||
...new Set(
|
||||
[...appOriginByOrigin.values()].filter(
|
||||
(o): o is string => o !== null,
|
||||
),
|
||||
),
|
||||
];
|
||||
const blockedByAppOrigin = new Map<string, boolean>();
|
||||
await Promise.all(
|
||||
uniqueAppOrigins.map(async (appOrigin) => {
|
||||
try {
|
||||
const block =
|
||||
await this.services.appOriginBlocklist.isOriginBlocked(
|
||||
appOrigin,
|
||||
);
|
||||
blockedByAppOrigin.set(appOrigin, block.blocked);
|
||||
} catch {
|
||||
blockedByAppOrigin.set(appOrigin, true);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
const resolvableAppOrigins = uniqueAppOrigins.filter(
|
||||
(appOrigin) => !blockedByAppOrigin.get(appOrigin),
|
||||
);
|
||||
const canonicalByAppOrigin =
|
||||
await this.#findCanonicalAppUidsForOrigins(resolvableAppOrigins);
|
||||
|
||||
// Gen-0 derived uid per app origin that missed the canonical lookup,
|
||||
// prefetched in one batch so `#derivedAppUidForOrigin`'s loop doesn't
|
||||
// pay a round trip for the common (no repoint) case.
|
||||
const missedAppOrigins = resolvableAppOrigins.filter(
|
||||
(appOrigin) => !canonicalByAppOrigin.get(appOrigin),
|
||||
);
|
||||
const gen0ByAppOrigin = new Map<string, string>();
|
||||
for (const appOrigin of missedAppOrigins) {
|
||||
gen0ByAppOrigin.set(appOrigin, this.#originUid(appOrigin, 0));
|
||||
}
|
||||
const gen0Uids = [...new Set(gen0ByAppOrigin.values())];
|
||||
const prefetched =
|
||||
gen0Uids.length > 0
|
||||
? await this.stores.app.getByUids(gen0Uids)
|
||||
: new Map<string, { index_url?: unknown }>();
|
||||
|
||||
const derivedByAppOrigin = new Map<string, string>();
|
||||
for (const appOrigin of missedAppOrigins) {
|
||||
const gen0Uid = gen0ByAppOrigin.get(appOrigin)!;
|
||||
const uid = await this.#derivedAppUidForOrigin(appOrigin, (uid) =>
|
||||
uid === gen0Uid
|
||||
? Promise.resolve(prefetched.get(uid) ?? null)
|
||||
: this.stores.app.getByUid(uid),
|
||||
);
|
||||
derivedByAppOrigin.set(appOrigin, uid);
|
||||
}
|
||||
|
||||
for (const origin of origins) {
|
||||
const appOrigin = appOriginByOrigin.get(origin) ?? null;
|
||||
if (appOrigin === null || blockedByAppOrigin.get(appOrigin)) {
|
||||
result.set(origin, null);
|
||||
continue;
|
||||
}
|
||||
const canonicalUid = canonicalByAppOrigin.get(appOrigin);
|
||||
result.set(
|
||||
origin,
|
||||
canonicalUid ?? derivedByAppOrigin.get(appOrigin) ?? null,
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalized origin of `url` with aliased hosts and hosting-domain variants
|
||||
* collapsed, after `app.from-origin` listeners have rewritten it. Null when
|
||||
@@ -834,17 +930,29 @@ export class AuthService extends PuterService {
|
||||
return event.origin;
|
||||
}
|
||||
|
||||
/** `app-<uuidv5(origin)>` for generation `gen` (0 is the first-visit uid). */
|
||||
#originUid(origin: string, gen: number): string {
|
||||
const name = gen === 0 ? origin : `${origin}#${gen}`;
|
||||
return `app-${uuidv5(name, APP_ORIGIN_UUID_NAMESPACE)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* `app-<uuidv5(origin)>`, unless that uid's row now lives at another
|
||||
* origin. A repointed row keeps its uid (and the data and grants keyed to
|
||||
* it), so the origin it left moves on to a successor uid instead of
|
||||
* resolving to someone's app it no longer serves.
|
||||
* resolving to someone's app it no longer serves. `getByUid` is injectable
|
||||
* so a batch caller can serve prefetched rows.
|
||||
*/
|
||||
async #derivedAppUidForOrigin(origin: string): Promise<string> {
|
||||
async #derivedAppUidForOrigin(
|
||||
origin: string,
|
||||
getByUid: (
|
||||
uid: string,
|
||||
) => Promise<{ index_url?: unknown } | null | undefined> = (uid) =>
|
||||
this.stores.app.getByUid(uid),
|
||||
): Promise<string> {
|
||||
for (let gen = 0; gen <= MAX_ORIGIN_UID_GENERATIONS; gen++) {
|
||||
const name = gen === 0 ? origin : `${origin}#${gen}`;
|
||||
const uid = `app-${uuidv5(name, APP_ORIGIN_UUID_NAMESPACE)}`;
|
||||
const app = await this.stores.app.getByUid(uid);
|
||||
const uid = this.#originUid(origin, gen);
|
||||
const app = await getByUid(uid);
|
||||
if (!app) return uid;
|
||||
if (
|
||||
typeof app.index_url === 'string' &&
|
||||
@@ -1058,21 +1166,17 @@ export class AuthService extends PuterService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the real app row whose `index_url` canonically matches `origin`.
|
||||
*
|
||||
* Build candidate URLs from the origin's subdomain crossed with every
|
||||
* configured hosting domain (static + private, with and without ports).
|
||||
* Prefer private rows, then the oldest match, for deterministic
|
||||
* tie-breaking across historically-duplicated rows.
|
||||
* Every `index_url` string that would canonically match `origin` — the
|
||||
* origin's subdomain crossed with every configured hosting domain (static
|
||||
* and private, with and without ports), alias-group hosts, and protocol
|
||||
* variants.
|
||||
*/
|
||||
async #findCanonicalAppUidForOrigin(
|
||||
origin: string,
|
||||
): Promise<string | null> {
|
||||
#indexUrlCandidatesForOrigin(origin: string): string[] {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(origin);
|
||||
} catch {
|
||||
return null;
|
||||
return [];
|
||||
}
|
||||
|
||||
const config = this.config as { protocol?: string };
|
||||
@@ -1120,18 +1224,103 @@ export class AuthService extends PuterService {
|
||||
urlCandidates.push(base, `${base}/`, `${base}/index.html`);
|
||||
}
|
||||
}
|
||||
const uniqueCandidates = [...new Set(urlCandidates)];
|
||||
if (uniqueCandidates.length === 0) return null;
|
||||
return [...new Set(urlCandidates)];
|
||||
}
|
||||
|
||||
const placeholders = uniqueCandidates.map(() => '?').join(', ');
|
||||
// Private rows win over public duplicates; oldest id breaks ties.
|
||||
const rows = (await this.clients.db.read(
|
||||
`SELECT \`uid\` FROM \`apps\` WHERE \`index_url\` IN (${placeholders}) ` +
|
||||
`ORDER BY CASE WHEN \`is_private\` = ${this.clients.db.booleanLiteral(true)} THEN 0 ELSE 1 END, \`id\` ASC LIMIT 1`,
|
||||
uniqueCandidates,
|
||||
)) as Array<{ uid?: string }>;
|
||||
const uid = rows[0]?.uid;
|
||||
return typeof uid === 'string' && uid ? uid : null;
|
||||
/**
|
||||
* Uid of the real app row whose `index_url` canonically matches `origin`;
|
||||
* private rows first, then the oldest, across historical duplicates.
|
||||
*/
|
||||
async #findCanonicalAppUidForOrigin(
|
||||
origin: string,
|
||||
): Promise<string | null> {
|
||||
return this.stores.app.findCanonicalUidByIndexUrlCandidates(
|
||||
this.#indexUrlCandidatesForOrigin(origin),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Batched {@link #findCanonicalAppUidForOrigin}, keyed by the origins passed
|
||||
* in, with the same private-first, oldest-id rule.
|
||||
*/
|
||||
async #findCanonicalAppUidsForOrigins(
|
||||
origins: string[],
|
||||
): Promise<Map<string, string | null>> {
|
||||
const result = new Map<string, string | null>();
|
||||
const candidateSetByOrigin = new Map<string, Set<string>>();
|
||||
const allCandidates = new Set<string>();
|
||||
|
||||
for (const origin of origins) {
|
||||
const candidates = this.#indexUrlCandidatesForOrigin(origin);
|
||||
candidateSetByOrigin.set(
|
||||
origin,
|
||||
new Set(candidates.map((c) => c.toLowerCase())),
|
||||
);
|
||||
for (const c of candidates) allCandidates.add(c);
|
||||
}
|
||||
|
||||
if (allCandidates.size === 0) {
|
||||
for (const origin of origins) result.set(origin, null);
|
||||
return result;
|
||||
}
|
||||
|
||||
const winners = (await this.stores.app.listIndexUrlWinners([
|
||||
...allCandidates,
|
||||
])) as Array<{
|
||||
index_url: unknown;
|
||||
private_id: unknown;
|
||||
min_id: unknown;
|
||||
}>;
|
||||
const winnerByIndexUrl = new Map<string, (typeof winners)[number]>();
|
||||
for (const winner of winners) {
|
||||
if (typeof winner.index_url === 'string') {
|
||||
winnerByIndexUrl.set(winner.index_url.toLowerCase(), winner);
|
||||
}
|
||||
}
|
||||
|
||||
// Lowest private id across the origin's matching groups, else lowest id.
|
||||
const winnerIdByOrigin = new Map<string, number>();
|
||||
for (const origin of origins) {
|
||||
const candidateSet = candidateSetByOrigin.get(origin);
|
||||
if (!candidateSet) continue;
|
||||
|
||||
let bestPrivateId: number | null = null;
|
||||
let bestMinId: number | null = null;
|
||||
for (const candidate of candidateSet) {
|
||||
const winner = winnerByIndexUrl.get(candidate);
|
||||
if (!winner) continue;
|
||||
if (winner.private_id != null) {
|
||||
const id = Number(winner.private_id);
|
||||
if (bestPrivateId === null || id < bestPrivateId) {
|
||||
bestPrivateId = id;
|
||||
}
|
||||
}
|
||||
if (winner.min_id != null) {
|
||||
const id = Number(winner.min_id);
|
||||
if (bestMinId === null || id < bestMinId) {
|
||||
bestMinId = id;
|
||||
}
|
||||
}
|
||||
}
|
||||
const winnerId = bestPrivateId ?? bestMinId;
|
||||
if (winnerId !== null) winnerIdByOrigin.set(origin, winnerId);
|
||||
}
|
||||
|
||||
const uniqueIds = [...new Set(winnerIdByOrigin.values())];
|
||||
// Aggregates can come back as strings; they were Number()-ed above to
|
||||
// match `getByIds` keys.
|
||||
const appsById =
|
||||
uniqueIds.length > 0
|
||||
? await this.stores.app.getByIds(uniqueIds)
|
||||
: new Map<number, { uid?: unknown }>();
|
||||
|
||||
for (const origin of origins) {
|
||||
const id = winnerIdByOrigin.get(origin);
|
||||
const uid = id !== undefined ? appsById.get(id)?.uid : undefined;
|
||||
result.set(origin, typeof uid === 'string' && uid ? uid : null);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async getUserAppToken(actor: Actor, appUid: string): Promise<string> {
|
||||
|
||||
@@ -58,6 +58,8 @@ const OLD_APP_NAME_TTL_MONTHS = 3;
|
||||
// limit is 999; staying well under that keeps `getByIds` portable across
|
||||
// backends without splitting the cap by driver.
|
||||
const BULK_QUERY_CHUNK_SIZE = 200;
|
||||
// Placeholders per index_url `IN (…)` chunk; under SQLite's old 999 default.
|
||||
const INDEX_URL_CHUNK_SIZE = 900;
|
||||
|
||||
// Top-level all-time open/user counts: hot path, slow to compute. Cached
|
||||
// lazily on read (pipelined MGET on every app list/read; misses query the
|
||||
@@ -299,15 +301,105 @@ export class AppStore extends PuterStore {
|
||||
/**
|
||||
* Every app whose `index_url` matches one of `candidates`, with the owner
|
||||
* and the app that built it. Used by the subdomain driver to decide who may
|
||||
* re-create a hosted name that apps still point at.
|
||||
* re-create a hosted name that apps still point at, and to derive a
|
||||
* subdomain row's associated app.
|
||||
*/
|
||||
async listByIndexUrlCandidates(candidates) {
|
||||
if (!Array.isArray(candidates) || candidates.length === 0) return [];
|
||||
const placeholders = candidates.map(() => '?').join(', ');
|
||||
return this.clients.db.read(
|
||||
`SELECT \`id\`, \`uid\`, \`owner_user_id\`, \`app_owner\` FROM \`apps\` WHERE \`index_url\` IN (${placeholders})`,
|
||||
[...candidates],
|
||||
const uniqueCandidates = [
|
||||
...new Set(
|
||||
(Array.isArray(candidates) ? candidates : []).filter(
|
||||
(c) => typeof c === 'string' && c.length > 0,
|
||||
),
|
||||
),
|
||||
];
|
||||
if (uniqueCandidates.length === 0) return [];
|
||||
|
||||
const rowsById = new Map();
|
||||
for (
|
||||
let offset = 0;
|
||||
offset < uniqueCandidates.length;
|
||||
offset += INDEX_URL_CHUNK_SIZE
|
||||
) {
|
||||
const chunk = uniqueCandidates.slice(
|
||||
offset,
|
||||
offset + INDEX_URL_CHUNK_SIZE,
|
||||
);
|
||||
const placeholders = chunk.map(() => '?').join(', ');
|
||||
const rows = await this.clients.db.read(
|
||||
`SELECT \`id\`, \`uid\`, \`owner_user_id\`, \`app_owner\`, \`index_url\` FROM \`apps\` WHERE \`index_url\` IN (${placeholders})`,
|
||||
chunk,
|
||||
);
|
||||
for (const row of rows) rowsById.set(row.id, row);
|
||||
}
|
||||
return [...rowsById.values()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical app uid among rows matching `candidates`: private first, then
|
||||
* oldest.
|
||||
*/
|
||||
async findCanonicalUidByIndexUrlCandidates(candidates) {
|
||||
const uniqueCandidates = [
|
||||
...new Set(
|
||||
(Array.isArray(candidates) ? candidates : []).filter(
|
||||
(c) => typeof c === 'string' && c.length > 0,
|
||||
),
|
||||
),
|
||||
];
|
||||
if (uniqueCandidates.length === 0) return null;
|
||||
|
||||
const placeholders = uniqueCandidates.map(() => '?').join(', ');
|
||||
const rows = await this.clients.db.read(
|
||||
`SELECT \`uid\` FROM \`apps\` WHERE \`index_url\` IN (${placeholders}) ` +
|
||||
`ORDER BY CASE WHEN \`is_private\` = ${this.clients.db.booleanLiteral(true)} THEN 0 ELSE 1 END, \`id\` ASC LIMIT 1`,
|
||||
uniqueCandidates,
|
||||
);
|
||||
const uid = rows[0]?.uid;
|
||||
return typeof uid === 'string' && uid ? uid : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* One row per distinct `index_url` in `candidates` with its lowest private
|
||||
* id (`private_id`, null if none) and lowest id (`min_id`). An external
|
||||
* index_url can be shared by thousands of apps, so batch callers pick
|
||||
* canonical winners from these instead of every matching row.
|
||||
*/
|
||||
async listIndexUrlWinners(candidates) {
|
||||
const uniqueCandidates = [
|
||||
...new Set(
|
||||
(Array.isArray(candidates) ? candidates : []).filter(
|
||||
(c) => typeof c === 'string' && c.length > 0,
|
||||
),
|
||||
),
|
||||
];
|
||||
if (uniqueCandidates.length === 0) return [];
|
||||
|
||||
const winnersByIndexUrl = new Map();
|
||||
for (
|
||||
let offset = 0;
|
||||
offset < uniqueCandidates.length;
|
||||
offset += INDEX_URL_CHUNK_SIZE
|
||||
) {
|
||||
const chunk = uniqueCandidates.slice(
|
||||
offset,
|
||||
offset + INDEX_URL_CHUNK_SIZE,
|
||||
);
|
||||
const placeholders = chunk.map(() => '?').join(', ');
|
||||
const rows = await this.clients.db.read(
|
||||
`SELECT \`index_url\`, ` +
|
||||
`MIN(CASE WHEN \`is_private\` = ${this.clients.db.booleanLiteral(true)} THEN \`id\` END) AS private_id, ` +
|
||||
`MIN(\`id\`) AS min_id ` +
|
||||
`FROM \`apps\` WHERE \`index_url\` IN (${placeholders}) GROUP BY \`index_url\``,
|
||||
chunk,
|
||||
);
|
||||
// Deduped candidates land in one chunk each, so groups never span chunks.
|
||||
for (const row of rows) {
|
||||
if (typeof row.index_url === 'string') {
|
||||
winnersByIndexUrl.set(row.index_url, row);
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...winnersByIndexUrl.values()];
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1419,3 +1419,168 @@ describe('AppStore deleted-row tombstones', () => {
|
||||
expect(await redis.get(`apps:uid:${uid}`)).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('AppStore listByIndexUrlCandidates', () => {
|
||||
let server;
|
||||
let appStore;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await setupTestServer();
|
||||
appStore = server.stores.app;
|
||||
// Shared mock redis persists `apps:id:*` across servers while each
|
||||
// fresh sqlite db restarts its id sequence at 1 — without this a
|
||||
// freshly-created row can read back a stale cached row at the same id.
|
||||
await clearAppCache(server.clients.redis);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
});
|
||||
|
||||
const createApp = async (indexUrl) => {
|
||||
const name = `idxurl-${Math.random().toString(36).slice(2, 10)}`;
|
||||
return appStore.create(
|
||||
{ name, title: name, index_url: indexUrl },
|
||||
{ ownerUserId: 1 },
|
||||
);
|
||||
};
|
||||
|
||||
it('chunks a candidate list spanning more than one page and still finds the real URL at the end', async () => {
|
||||
const real = await createApp('https://real-target.example.com/');
|
||||
// 2,000 candidates is more than one 900-wide chunk; the real URL
|
||||
// sits past the first chunk boundary.
|
||||
const candidates = [];
|
||||
for (let i = 0; i < 2000; i++) {
|
||||
candidates.push(`https://decoy-${i}.example.com/`);
|
||||
}
|
||||
candidates.push(real.index_url);
|
||||
|
||||
const rows = await appStore.listByIndexUrlCandidates(candidates);
|
||||
expect(rows.map((r) => r.id)).toEqual([real.id]);
|
||||
});
|
||||
|
||||
it('dedupes a candidate repeated across chunk boundaries to one row', async () => {
|
||||
const real = await createApp('https://dup-target.example.com/');
|
||||
const candidates = Array.from({ length: 2000 }, () => real.index_url);
|
||||
|
||||
const rows = await appStore.listByIndexUrlCandidates(candidates);
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].id).toBe(real.id);
|
||||
});
|
||||
|
||||
it('returns [] for an empty or non-array candidate list', async () => {
|
||||
expect(await appStore.listByIndexUrlCandidates([])).toEqual([]);
|
||||
expect(await appStore.listByIndexUrlCandidates(null)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('AppStore findCanonicalUidByIndexUrlCandidates', () => {
|
||||
let server;
|
||||
let appStore;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await setupTestServer();
|
||||
appStore = server.stores.app;
|
||||
await clearAppCache(server.clients.redis);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
});
|
||||
|
||||
it('prefers the private row, then the oldest match', async () => {
|
||||
const url = `https://canon-${Math.random().toString(36).slice(2, 10)}.example.com/`;
|
||||
const pub = await appStore.create(
|
||||
{ name: `pub-${Math.random().toString(36).slice(2, 8)}`, title: 't', index_url: url },
|
||||
{ ownerUserId: 1 },
|
||||
);
|
||||
const privUid = `app-${Math.random().toString(36).slice(2, 10)}`;
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `is_private`) VALUES (?, ?, ?, ?, ?)',
|
||||
[privUid, `priv-${Math.random().toString(36).slice(2, 8)}`, 't', url, 1],
|
||||
);
|
||||
|
||||
const uid = await appStore.findCanonicalUidByIndexUrlCandidates([url]);
|
||||
expect(uid).toBe(privUid);
|
||||
expect(uid).not.toBe(pub.uid);
|
||||
});
|
||||
|
||||
it('returns null for an empty, non-array, or non-matching candidate list', async () => {
|
||||
expect(await appStore.findCanonicalUidByIndexUrlCandidates([])).toBeNull();
|
||||
expect(await appStore.findCanonicalUidByIndexUrlCandidates(null)).toBeNull();
|
||||
expect(
|
||||
await appStore.findCanonicalUidByIndexUrlCandidates([
|
||||
'https://nothing-here.example.com/',
|
||||
]),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('AppStore listIndexUrlWinners', () => {
|
||||
let server;
|
||||
let appStore;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await setupTestServer();
|
||||
appStore = server.stores.app;
|
||||
await clearAppCache(server.clients.redis);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
});
|
||||
|
||||
const createApp = async (indexUrl) => {
|
||||
const name = `winner-${Math.random().toString(36).slice(2, 10)}`;
|
||||
return appStore.create(
|
||||
{ name, title: name, index_url: indexUrl },
|
||||
{ ownerUserId: 1 },
|
||||
);
|
||||
};
|
||||
|
||||
it('collapses many apps sharing one index_url into a single winner row', async () => {
|
||||
// An external dev-server default — the exact shape that can be
|
||||
// shared by thousands of apps and would blow up a row-per-app query.
|
||||
const sharedUrl = `http://localhost:${5000 + Math.floor(Math.random() * 1000)}/`;
|
||||
for (let i = 0; i < 50; i++) {
|
||||
await createApp(sharedUrl);
|
||||
}
|
||||
const otherUrl = `https://distinct-${Math.random().toString(36).slice(2, 10)}.example.com/`;
|
||||
await createApp(otherUrl);
|
||||
|
||||
const rows = await appStore.listIndexUrlWinners([sharedUrl, otherUrl]);
|
||||
expect(rows.length).toBe(2);
|
||||
// One row per distinct index_url, never one per matching app.
|
||||
expect(rows.length).toBeLessThanOrEqual(2);
|
||||
const shared = rows.find((r) => r.index_url === sharedUrl);
|
||||
expect(shared).toBeTruthy();
|
||||
expect(shared.private_id == null).toBe(true);
|
||||
expect(Number(shared.min_id)).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('reports the lowest private id and the lowest id overall per index_url', async () => {
|
||||
const url = `https://winner-${Math.random().toString(36).slice(2, 10)}.example.com/`;
|
||||
const pub = await createApp(url);
|
||||
const priv1Uid = `app-${Math.random().toString(36).slice(2, 10)}`;
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `is_private`) VALUES (?, ?, ?, ?, ?)',
|
||||
[priv1Uid, `priv1-${Math.random().toString(36).slice(2, 8)}`, 't', url, 1],
|
||||
);
|
||||
const priv1 = await appStore.getByUid(priv1Uid);
|
||||
const priv2Uid = `app-${Math.random().toString(36).slice(2, 10)}`;
|
||||
await server.clients.db.write(
|
||||
'INSERT INTO `apps` (`uid`, `name`, `title`, `index_url`, `is_private`) VALUES (?, ?, ?, ?, ?)',
|
||||
[priv2Uid, `priv2-${Math.random().toString(36).slice(2, 8)}`, 't', url, 1],
|
||||
);
|
||||
const priv2 = await appStore.getByUid(priv2Uid);
|
||||
|
||||
const [winner] = await appStore.listIndexUrlWinners([url]);
|
||||
expect(Number(winner.min_id)).toBe(pub.id);
|
||||
expect(Number(winner.private_id)).toBe(Math.min(priv1.id, priv2.id));
|
||||
});
|
||||
|
||||
it('returns [] for an empty or non-array candidate list', async () => {
|
||||
expect(await appStore.listIndexUrlWinners([])).toEqual([]);
|
||||
expect(await appStore.listIndexUrlWinners(null)).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -563,3 +563,90 @@ describe('SubdomainStore reads and writes', () => {
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// -- getBySubdomains (batched) --
|
||||
|
||||
describe('SubdomainStore.getBySubdomains', () => {
|
||||
it('is keyed by the requested names and omits names with no row', async () => {
|
||||
const owner = await makeUser();
|
||||
const a = `gbs-a-${Math.random().toString(36).slice(2, 8)}`;
|
||||
const b = `gbs-b-${Math.random().toString(36).slice(2, 8)}`;
|
||||
const missing = `gbs-missing-${Math.random().toString(36).slice(2, 8)}`;
|
||||
await store.create({ userId: owner, subdomain: a } as never);
|
||||
await store.create({ userId: owner, subdomain: b } as never);
|
||||
|
||||
const found = await store.getBySubdomains([a, b, missing]);
|
||||
|
||||
expect(found.size).toBe(2);
|
||||
expect(found.get(a)?.subdomain).toBe(a);
|
||||
expect(found.get(b)?.subdomain).toBe(b);
|
||||
expect(found.has(missing)).toBe(false);
|
||||
});
|
||||
|
||||
it('serves a cache hit with no DB read', async () => {
|
||||
const owner = await makeUser();
|
||||
const name = `gbs-cached-${Math.random().toString(36).slice(2, 8)}`;
|
||||
// `create` writes through the cache, so the row is already warm.
|
||||
await store.create({ userId: owner, subdomain: name } as never);
|
||||
|
||||
const read = vi.spyOn(server.clients.db, 'read');
|
||||
const pread = vi.spyOn(server.clients.db, 'pread');
|
||||
try {
|
||||
const found = await store.getBySubdomains([name]);
|
||||
expect(found.get(name)?.subdomain).toBe(name);
|
||||
expect(read).not.toHaveBeenCalled();
|
||||
expect(pread).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
pread.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('a cached negative marker is absent from the result with no DB read', async () => {
|
||||
const name = `gbs-neg-${Math.random().toString(36).slice(2, 8)}`;
|
||||
// Poison the negative-cache marker first.
|
||||
expect(await store.getBySubdomains([name])).toEqual(new Map());
|
||||
|
||||
const read = vi.spyOn(server.clients.db, 'read');
|
||||
try {
|
||||
const found = await store.getBySubdomains([name]);
|
||||
expect(found.has(name)).toBe(false);
|
||||
expect(read).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('treats a cached JSON null as a miss and reads the DB, like getBySubdomain', async () => {
|
||||
const owner = await makeUser();
|
||||
const name = `gbs-null-${Math.random().toString(36).slice(2, 8)}`;
|
||||
await store.create({ userId: owner, subdomain: name } as never);
|
||||
await server.clients.redis.set(cacheKey(name), 'null');
|
||||
|
||||
const found = await store.getBySubdomains([name]);
|
||||
expect(found.get(name)?.subdomain).toBe(name);
|
||||
});
|
||||
|
||||
it('a primary read uses pread and writes through, healing a stale negative marker', async () => {
|
||||
const owner = await makeUser();
|
||||
const name = `gbs-heal-${Math.random().toString(36).slice(2, 8)}`;
|
||||
// Poison the cache with a negative marker, then create the row.
|
||||
expect(await store.getBySubdomains([name])).toEqual(new Map());
|
||||
await store.create({ userId: owner, subdomain: name } as never);
|
||||
|
||||
const pread = vi.spyOn(server.clients.db, 'pread');
|
||||
try {
|
||||
const primaryFound = await store.getBySubdomains([name], {
|
||||
primary: true,
|
||||
});
|
||||
expect(primaryFound.get(name)?.subdomain).toBe(name);
|
||||
expect(pread).toHaveBeenCalled();
|
||||
} finally {
|
||||
pread.mockRestore();
|
||||
}
|
||||
|
||||
// The write-through heals the replica-path cache too.
|
||||
const healedFound = await store.getBySubdomains([name]);
|
||||
expect(healedFound.get(name)?.subdomain).toBe(name);
|
||||
});
|
||||
});
|
||||
@@ -111,6 +111,8 @@ const CACHE_TTL_SECONDS = 60 * 60;
|
||||
// Sentinel so 404s on the same public subdomain don't hit the DB repeatedly.
|
||||
const NEGATIVE_CACHE_MARKER = '__none__';
|
||||
const NEGATIVE_CACHE_TTL_SECONDS = 10;
|
||||
// Cap on placeholders per `IN (?, ?, …)` chunk — mirrors AppStore/UserStore.
|
||||
const BULK_QUERY_CHUNK_SIZE = 200;
|
||||
|
||||
export class SubdomainStore extends PuterStore {
|
||||
// -- Reads --------------------------------------------------------
|
||||
@@ -170,34 +172,115 @@ export class SubdomainStore extends PuterStore {
|
||||
: await this.clients.db.read(sql, [subdomain]);
|
||||
const row = (rows[0] as unknown as SubdomainRow | undefined) ?? null;
|
||||
|
||||
// These writes are fire-and-forget, so a mutation that lands between
|
||||
// the SELECT above and the SET below would otherwise be overwritten
|
||||
// by the row we just read — stranding the pre-write row in cache for
|
||||
// the full TTL (an hour of a site serving its old root_dir after
|
||||
// `hosting.update`). A replica read is not authoritative, so it only
|
||||
// *populates* an absent key (`NX`) and can never clobber a fresher
|
||||
// write. A `primary` read is read-after-write on the primary, so it
|
||||
// writes through — that's what heals a stale negative marker.
|
||||
const populate = (value: string, ttlSeconds: number) =>
|
||||
(primary
|
||||
? this.clients.redis.set(cacheKey, value, 'EX', ttlSeconds)
|
||||
: this.clients.redis.set(
|
||||
cacheKey,
|
||||
value,
|
||||
'EX',
|
||||
ttlSeconds,
|
||||
'NX',
|
||||
)
|
||||
).catch(() => {});
|
||||
|
||||
if (row) {
|
||||
populate(JSON.stringify(row), CACHE_TTL_SECONDS);
|
||||
void this.#populateCache(
|
||||
cacheKey,
|
||||
JSON.stringify(row),
|
||||
CACHE_TTL_SECONDS,
|
||||
primary,
|
||||
);
|
||||
} else {
|
||||
populate(NEGATIVE_CACHE_MARKER, NEGATIVE_CACHE_TTL_SECONDS);
|
||||
void this.#populateCache(
|
||||
cacheKey,
|
||||
NEGATIVE_CACHE_MARKER,
|
||||
NEGATIVE_CACHE_TTL_SECONDS,
|
||||
primary,
|
||||
);
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
/**
|
||||
* Batched sibling of {@link getBySubdomain} — one cache pipeline plus one
|
||||
* chunked `IN (…)` query for the misses, instead of one lookup per name.
|
||||
* Keyed by the requested names; a name with no row (negative-cached or
|
||||
* genuinely absent) is simply missing from the returned map.
|
||||
*/
|
||||
async getBySubdomains(
|
||||
names: string[],
|
||||
{ primary = false }: { primary?: boolean } = {},
|
||||
): Promise<Map<string, SubdomainRow>> {
|
||||
const result = new Map<string, SubdomainRow>();
|
||||
const uniqueNames = [
|
||||
...new Set(
|
||||
(Array.isArray(names) ? names : []).filter(
|
||||
(n): n is string => typeof n === 'string' && n.length > 0,
|
||||
),
|
||||
),
|
||||
];
|
||||
if (uniqueNames.length === 0) return result;
|
||||
|
||||
let missingNames = uniqueNames;
|
||||
if (!primary) {
|
||||
missingNames = [];
|
||||
try {
|
||||
const pipeline = this.clients.redis.pipeline();
|
||||
for (const name of uniqueNames) {
|
||||
pipeline.get(this.#cacheKey(name));
|
||||
}
|
||||
const cacheResults = (await pipeline.exec()) ?? [];
|
||||
for (let i = 0; i < uniqueNames.length; i++) {
|
||||
const name = uniqueNames[i]!;
|
||||
const raw = cacheResults[i]?.[1];
|
||||
if (raw === NEGATIVE_CACHE_MARKER) continue; // cached miss
|
||||
if (typeof raw === 'string') {
|
||||
try {
|
||||
const parsed = JSON.parse(
|
||||
raw,
|
||||
) as SubdomainRow | null;
|
||||
if (parsed) {
|
||||
result.set(name, parsed);
|
||||
continue;
|
||||
}
|
||||
} catch {
|
||||
// Fall through to DB on any parse failure.
|
||||
}
|
||||
}
|
||||
missingNames.push(name);
|
||||
}
|
||||
} catch {
|
||||
missingNames = uniqueNames;
|
||||
}
|
||||
}
|
||||
if (missingNames.length === 0) return result;
|
||||
|
||||
const rowsByName = new Map<string, SubdomainRow>();
|
||||
for (
|
||||
let offset = 0;
|
||||
offset < missingNames.length;
|
||||
offset += BULK_QUERY_CHUNK_SIZE
|
||||
) {
|
||||
const chunk = missingNames.slice(
|
||||
offset,
|
||||
offset + BULK_QUERY_CHUNK_SIZE,
|
||||
);
|
||||
const placeholders = chunk.map(() => '?').join(', ');
|
||||
const sql = `SELECT * FROM \`subdomains\` WHERE \`subdomain\` IN (${placeholders})`;
|
||||
const rows = (primary
|
||||
? await this.clients.db.pread(sql, chunk)
|
||||
: await this.clients.db.read(
|
||||
sql,
|
||||
chunk,
|
||||
)) as unknown as SubdomainRow[];
|
||||
// Keyed lowercase: a case-insensitive collation can return a row
|
||||
// cased differently from the name, which `getBySubdomain` accepts.
|
||||
for (const row of rows) {
|
||||
if (typeof row.subdomain !== 'string') continue;
|
||||
const key = row.subdomain.toLowerCase();
|
||||
if (!rowsByName.has(key)) rowsByName.set(key, row);
|
||||
}
|
||||
}
|
||||
|
||||
for (const name of missingNames) {
|
||||
const row = rowsByName.get(name.toLowerCase());
|
||||
if (row) result.set(name, row);
|
||||
}
|
||||
|
||||
void this.#populateCacheForNames(missingNames, result, primary);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async listByUserId(
|
||||
userId: number,
|
||||
{
|
||||
@@ -634,6 +717,61 @@ export class SubdomainStore extends PuterStore {
|
||||
return `${CACHE_KEY_PREFIX}:listByUserPrefixKeys:${userId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write-through on a primary read (heals a stale negative marker); `NX` on
|
||||
* a replica read so an in-flight fresher write can't be clobbered by a
|
||||
* stale value landing after it.
|
||||
*/
|
||||
async #populateCache(
|
||||
cacheKey: string,
|
||||
value: string,
|
||||
ttlSeconds: number,
|
||||
primary: boolean,
|
||||
): Promise<void> {
|
||||
try {
|
||||
if (primary) {
|
||||
await this.clients.redis.set(cacheKey, value, 'EX', ttlSeconds);
|
||||
} else {
|
||||
await this.clients.redis.set(
|
||||
cacheKey,
|
||||
value,
|
||||
'EX',
|
||||
ttlSeconds,
|
||||
'NX',
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
}
|
||||
|
||||
/** Pipelined sibling of {@link #populateCache} for `getBySubdomains`. */
|
||||
async #populateCacheForNames(
|
||||
names: string[],
|
||||
rowsByName: Map<string, SubdomainRow>,
|
||||
primary: boolean,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const pipeline = this.clients.redis.pipeline();
|
||||
for (const name of names) {
|
||||
const row = rowsByName.get(name);
|
||||
const cacheKey = this.#cacheKey(name);
|
||||
const value = row ? JSON.stringify(row) : NEGATIVE_CACHE_MARKER;
|
||||
const ttl = row
|
||||
? CACHE_TTL_SECONDS
|
||||
: NEGATIVE_CACHE_TTL_SECONDS;
|
||||
if (primary) {
|
||||
pipeline.set(cacheKey, value, 'EX', ttl);
|
||||
} else {
|
||||
pipeline.set(cacheKey, value, 'EX', ttl, 'NX');
|
||||
}
|
||||
}
|
||||
await pipeline.exec();
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
}
|
||||
|
||||
async #refreshCache(row: { subdomain?: string }) {
|
||||
if (!row?.subdomain) return;
|
||||
await this.publishCacheKeys({
|
||||
|
||||
@@ -17,13 +17,17 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { PuterServer } from '../server.js';
|
||||
import { setupTestServer } from '../testUtil.js';
|
||||
import {
|
||||
buildHostedBackingDenial,
|
||||
buildHostedSubdomainIndexUrlCandidates,
|
||||
extractPuterHostedSubdomain,
|
||||
getPuterHostedDomains,
|
||||
hostedIndexUrlBackingIsUnavailable,
|
||||
hostedIndexUrlBackingsAreUnavailable,
|
||||
} from './hostedAppBacking.js';
|
||||
|
||||
// Pure-unit companion to the integration coverage in
|
||||
@@ -354,3 +358,159 @@ describe('buildHostedBackingDenial', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// Matches the hosting domains in config.default.json, so subdomain rows
|
||||
// created through the real store resolve the same way `AppDriver` would see
|
||||
// them in production.
|
||||
const HOSTING_CONFIG = {
|
||||
static_hosting_domain: 'site.puter.localhost',
|
||||
static_hosting_domain_alt: 'host.puter.localhost',
|
||||
private_app_hosting_domain: 'app.puter.localhost',
|
||||
private_app_hosting_domain_alt: 'dev.puter.localhost',
|
||||
};
|
||||
const hostedUrl = (sub: string) => `https://${sub}.site.puter.localhost/`;
|
||||
|
||||
describe('hostedIndexUrlBackingsAreUnavailable (batched, against a real store)', () => {
|
||||
let server: PuterServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
server = await setupTestServer();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
});
|
||||
|
||||
const makeUser = async () => {
|
||||
const username = `hab-${Math.random().toString(36).slice(2, 10)}`;
|
||||
return server.stores.user.create({
|
||||
username,
|
||||
uuid: uuidv4(),
|
||||
password: null,
|
||||
email: `${username}@test.local`,
|
||||
free_storage: 100 * 1024 * 1024,
|
||||
requires_email_confirmation: false,
|
||||
});
|
||||
};
|
||||
|
||||
it('resolves a mix of apps in input order with one batched call', async () => {
|
||||
const owner = await makeUser();
|
||||
const attacker = await makeUser();
|
||||
|
||||
const liveSub = `live-${Math.random().toString(36).slice(2, 8)}`;
|
||||
await server.stores.subdomain.create({
|
||||
userId: owner.id,
|
||||
subdomain: liveSub,
|
||||
});
|
||||
const reclaimedSub = `reclaimed-${Math.random().toString(36).slice(2, 8)}`;
|
||||
await server.stores.subdomain.create({
|
||||
userId: attacker.id,
|
||||
subdomain: reclaimedSub,
|
||||
});
|
||||
const danglingSub = `dangling-${Math.random().toString(36).slice(2, 8)}`;
|
||||
|
||||
const external = {
|
||||
index_url: 'https://elsewhere.example.com/',
|
||||
owner_user_id: owner.id,
|
||||
};
|
||||
const live = { index_url: hostedUrl(liveSub), owner_user_id: owner.id };
|
||||
const reclaimed = {
|
||||
index_url: hostedUrl(reclaimedSub),
|
||||
owner_user_id: owner.id,
|
||||
};
|
||||
const dangling = {
|
||||
index_url: hostedUrl(danglingSub),
|
||||
owner_user_id: owner.id,
|
||||
};
|
||||
|
||||
const getBySubdomains = vi.spyOn(
|
||||
server.stores.subdomain,
|
||||
'getBySubdomains',
|
||||
);
|
||||
try {
|
||||
const results = await hostedIndexUrlBackingsAreUnavailable({
|
||||
apps: [external, live, reclaimed, dangling],
|
||||
subdomainStore: server.stores.subdomain,
|
||||
config: HOSTING_CONFIG,
|
||||
});
|
||||
// Input order preserved; external → false, live → false, a
|
||||
// reclaimed owner → true, a gone subdomain → true.
|
||||
expect(results).toEqual([false, false, true, true]);
|
||||
|
||||
// One non-primary batch carrying only the three hosted names —
|
||||
// the external app never touches the store.
|
||||
expect(getBySubdomains).toHaveBeenCalledTimes(2);
|
||||
expect([...getBySubdomains.mock.calls[0]![0]].sort()).toEqual(
|
||||
[liveSub, reclaimedSub, danglingSub].sort(),
|
||||
);
|
||||
// Primary follow-up only for the one name still missing.
|
||||
expect(getBySubdomains.mock.calls[1]).toEqual([
|
||||
[danglingSub],
|
||||
{ primary: true },
|
||||
]);
|
||||
} finally {
|
||||
getBySubdomains.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('makes no store call when every app is non-hosted', async () => {
|
||||
const owner = await makeUser();
|
||||
const external = {
|
||||
index_url: 'https://elsewhere.example.com/',
|
||||
owner_user_id: owner.id,
|
||||
};
|
||||
const builtin = { index_url: undefined, owner_user_id: owner.id };
|
||||
|
||||
const getBySubdomains = vi.spyOn(
|
||||
server.stores.subdomain,
|
||||
'getBySubdomains',
|
||||
);
|
||||
try {
|
||||
const results = await hostedIndexUrlBackingsAreUnavailable({
|
||||
apps: [external, builtin],
|
||||
subdomainStore: server.stores.subdomain,
|
||||
config: HOSTING_CONFIG,
|
||||
});
|
||||
expect(results).toEqual([false, false]);
|
||||
expect(getBySubdomains).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
getBySubdomains.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('does exactly one primary batch covering every still-missing name', async () => {
|
||||
const owner = await makeUser();
|
||||
const goneA = `gone-a-${Math.random().toString(36).slice(2, 6)}`;
|
||||
const goneB = `gone-b-${Math.random().toString(36).slice(2, 6)}`;
|
||||
const danglingA = {
|
||||
index_url: hostedUrl(goneA),
|
||||
owner_user_id: owner.id,
|
||||
};
|
||||
const danglingB = {
|
||||
index_url: hostedUrl(goneB),
|
||||
owner_user_id: owner.id,
|
||||
};
|
||||
|
||||
const getBySubdomains = vi.spyOn(
|
||||
server.stores.subdomain,
|
||||
'getBySubdomains',
|
||||
);
|
||||
try {
|
||||
const results = await hostedIndexUrlBackingsAreUnavailable({
|
||||
apps: [danglingA, danglingB],
|
||||
subdomainStore: server.stores.subdomain,
|
||||
config: HOSTING_CONFIG,
|
||||
});
|
||||
expect(results).toEqual([true, true]);
|
||||
expect(getBySubdomains).toHaveBeenCalledTimes(2);
|
||||
expect(getBySubdomains.mock.calls[1]![1]).toEqual({
|
||||
primary: true,
|
||||
});
|
||||
expect([...getBySubdomains.mock.calls[1]![0]].sort()).toEqual(
|
||||
[goneA, goneB].sort(),
|
||||
);
|
||||
} finally {
|
||||
getBySubdomains.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -181,6 +181,24 @@ export function extractPuterHostedSubdomain(
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when `row` (the subdomain backing `app`'s hosted index_url, or null)
|
||||
* means the app's launch is unsafe: gone, or reclaimed by a different owner.
|
||||
*/
|
||||
function backingRowIsUnavailable(
|
||||
app: AppBackingRow,
|
||||
row: { user_id?: unknown } | null,
|
||||
): boolean {
|
||||
if (!row) return true; // subdomain no longer exists → dangling
|
||||
|
||||
const appOwnerId = Number(app.owner_user_id);
|
||||
const subdomainOwnerId = Number(row.user_id);
|
||||
if (!Number.isInteger(appOwnerId) || !Number.isInteger(subdomainOwnerId)) {
|
||||
return true;
|
||||
}
|
||||
return subdomainOwnerId !== appOwnerId;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the app's puter-hosted subdomain is missing, or is currently owned
|
||||
* by a different user than the app's owner (it was reclaimed — launching would
|
||||
@@ -208,14 +226,46 @@ export async function hostedIndexUrlBackingIsUnavailable({
|
||||
primary: true,
|
||||
});
|
||||
}
|
||||
if (!row) return true; // subdomain no longer exists → dangling
|
||||
return backingRowIsUnavailable(app, row);
|
||||
}
|
||||
|
||||
const appOwnerId = Number(app.owner_user_id);
|
||||
const subdomainOwnerId = Number(row.user_id);
|
||||
if (!Number.isInteger(appOwnerId) || !Number.isInteger(subdomainOwnerId)) {
|
||||
return true;
|
||||
/**
|
||||
* Batched sibling of {@link hostedIndexUrlBackingIsUnavailable} — one
|
||||
* `getBySubdomains` call (plus a single primary follow-up for whatever's still
|
||||
* missing) instead of per-app round trips. Returns a boolean per `apps` entry,
|
||||
* in input order, with the same semantics as the single-app function above.
|
||||
*/
|
||||
export async function hostedIndexUrlBackingsAreUnavailable({
|
||||
apps,
|
||||
subdomainStore,
|
||||
config,
|
||||
}: {
|
||||
apps: AppBackingRow[];
|
||||
subdomainStore: Pick<SubdomainStore, 'getBySubdomains'>;
|
||||
config: HostedDomainConfig | undefined | null;
|
||||
}): Promise<boolean[]> {
|
||||
const subdomains = apps.map((app) =>
|
||||
extractPuterHostedSubdomain(app.index_url, config),
|
||||
);
|
||||
const names = [
|
||||
...new Set(subdomains.filter((s): s is string => s !== null)),
|
||||
];
|
||||
if (names.length === 0) return apps.map(() => false);
|
||||
|
||||
let rowsByName = await subdomainStore.getBySubdomains(names);
|
||||
const missing = names.filter((name) => !rowsByName.has(name));
|
||||
if (missing.length > 0) {
|
||||
const primaryRows = await subdomainStore.getBySubdomains(missing, {
|
||||
primary: true,
|
||||
});
|
||||
rowsByName = new Map([...rowsByName, ...primaryRows]);
|
||||
}
|
||||
return subdomainOwnerId !== appOwnerId;
|
||||
|
||||
return apps.map((app, i) => {
|
||||
const subdomain = subdomains[i];
|
||||
if (!subdomain) return false;
|
||||
return backingRowIsUnavailable(app, rowsByName.get(subdomain) ?? null);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in new issue
Block a user