diff --git a/src/backend/clients/database/SqliteDatabaseClient.test.ts b/src/backend/clients/database/SqliteDatabaseClient.test.ts index d890246ec..0cb4d5461 100644 --- a/src/backend/clients/database/SqliteDatabaseClient.test.ts +++ b/src/backend/clients/database/SqliteDatabaseClient.test.ts @@ -27,7 +27,7 @@ import { DatabaseClientFactory } from './index.js'; import { SqliteDatabaseClient } from './SqliteDatabaseClient.js'; /** Highest schema version the migration table can reach. */ -const CURRENT_SCHEMA_VERSION = 69; +const CURRENT_SCHEMA_VERSION = 70; /** * These suites migrate real files on disk. Idle they finish in well under a diff --git a/src/backend/clients/database/SqliteDatabaseClient.ts b/src/backend/clients/database/SqliteDatabaseClient.ts index 4a37f0737..31dc1f362 100644 --- a/src/backend/clients/database/SqliteDatabaseClient.ts +++ b/src/backend/clients/database/SqliteDatabaseClient.ts @@ -103,6 +103,7 @@ const AVAILABLE_MIGRATIONS: [number, string[]][] = [ [66, ['0071_share_issuer_index.sql']], [67, ['0072_notification-scope.sql']], [68, ['0073_notification-created-at.sql']], + [69, ['0074_add_user_home.sql']], ]; export class SqliteDatabaseClient extends AbstractDatabaseClient { diff --git a/src/backend/clients/database/migrations/mysql/mysql_mig_1.sql b/src/backend/clients/database/migrations/mysql/mysql_mig_1.sql index d9306cbdb..23d50e8e7 100644 --- a/src/backend/clients/database/migrations/mysql/mysql_mig_1.sql +++ b/src/backend/clients/database/migrations/mysql/mysql_mig_1.sql @@ -1060,6 +1060,7 @@ CREATE TABLE IF NOT EXISTS `user` ( `signup_server` varchar(255) DEFAULT NULL, `metadata` json DEFAULT (json_object()), `reputation` smallint DEFAULT '100', + `home` varchar(64) DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `uid` (`uuid`), UNIQUE KEY `username` (`username`), @@ -1151,6 +1152,7 @@ CALL _puter_add_col('user', 'signup_user_agent', '`signup_user_agent` varchar(51 CALL _puter_add_col('user', 'signup_origin', '`signup_origin` varchar(255) DEFAULT NULL'); CALL _puter_add_col('user', 'signup_server', '`signup_server` varchar(255) DEFAULT NULL'); CALL _puter_add_col('user', 'metadata', '`metadata` json DEFAULT (json_object())'); +CALL _puter_add_col('user', 'home', '`home` varchar(64) DEFAULT NULL'); CALL _puter_add_col('user', 'reputation', '`reputation` smallint DEFAULT ''100'''); -- diff --git a/src/backend/clients/database/migrations/mysql/mysql_mig_28.sql b/src/backend/clients/database/migrations/mysql/mysql_mig_28.sql new file mode 100644 index 000000000..e9cf6f2cd --- /dev/null +++ b/src/backend/clients/database/migrations/mysql/mysql_mig_28.sql @@ -0,0 +1,25 @@ +-- Copyright (C) 2024-present Puter Technologies Inc. +-- +-- This file is part of Puter. +-- +-- Puter is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as published +-- by the Free Software Foundation, either version 3 of the License, or +-- (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU Affero General Public License for more details. +-- +-- You should have received a copy of the GNU Affero General Public License +-- along with this program. If not, see . + +-- Home region column. Mirrors SQLite migration 0074. Home region this +-- account's data belongs in — a node id, not a storage region. NULL for +-- accounts that predate the column, which fall back to `signup_server`. +-- +-- Idempotent: the column add uses _puter_add_col (defined in mig_1, which +-- leaves it resident for later migrations). + +CALL _puter_add_col('user', 'home', '`home` VARCHAR(64) DEFAULT NULL'); diff --git a/src/backend/clients/database/migrations/postgres/postgres_mig_1.sql b/src/backend/clients/database/migrations/postgres/postgres_mig_1.sql index 177d4475d..5fef6edf5 100644 --- a/src/backend/clients/database/migrations/postgres/postgres_mig_1.sql +++ b/src/backend/clients/database/migrations/postgres/postgres_mig_1.sql @@ -72,7 +72,8 @@ CREATE TABLE IF NOT EXISTS "user" ( signup_origin varchar(255), signup_server varchar(255), metadata jsonb DEFAULT '{}'::jsonb, - reputation smallint DEFAULT 100 + reputation smallint DEFAULT 100, + home varchar(64) ); CREATE INDEX IF NOT EXISTS idx_user_email ON "user" (email); diff --git a/src/backend/clients/database/migrations/postgres/postgres_mig_17.sql b/src/backend/clients/database/migrations/postgres/postgres_mig_17.sql new file mode 100644 index 000000000..77216abd7 --- /dev/null +++ b/src/backend/clients/database/migrations/postgres/postgres_mig_17.sql @@ -0,0 +1,23 @@ +-- Copyright (C) 2024-present Puter Technologies Inc. +-- +-- This file is part of Puter. +-- +-- Puter is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as published +-- by the Free Software Foundation, either version 3 of the License, or +-- (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU Affero General Public License for more details. +-- +-- You should have received a copy of the GNU Affero General Public License +-- along with this program. If not, see . + +-- Home region column. Mirrors SQLite migration 0074. Home region this +-- account's data belongs in — a node id, not a storage region. NULL for +-- accounts that predate the column, which fall back to `signup_server`. +-- Idempotent via IF NOT EXISTS. + +ALTER TABLE "user" ADD COLUMN IF NOT EXISTS home varchar(64); diff --git a/src/backend/clients/database/migrations/sqlite/0074_add_user_home.sql b/src/backend/clients/database/migrations/sqlite/0074_add_user_home.sql new file mode 100644 index 000000000..8b9dc150e --- /dev/null +++ b/src/backend/clients/database/migrations/sqlite/0074_add_user_home.sql @@ -0,0 +1,22 @@ +-- Copyright (C) 2024-present Puter Technologies Inc. +-- +-- This file is part of Puter. +-- +-- Puter is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as published +-- by the Free Software Foundation, either version 3 of the License, or +-- (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU Affero General Public License for more details. +-- +-- You should have received a copy of the GNU Affero General Public License +-- along with this program. If not, see . + +-- Home region this account's data belongs in — a node id, not a storage +-- region. NULL for every account that predates the column, which falls back to +-- `signup_server`. What a name maps to is deployment config, so adding a +-- region needs no migration. +ALTER TABLE `user` ADD COLUMN `home` VARCHAR(64) DEFAULT NULL; diff --git a/src/backend/extensions.ts b/src/backend/extensions.ts index 24bc9db99..daa04c943 100644 --- a/src/backend/extensions.ts +++ b/src/backend/extensions.ts @@ -281,6 +281,15 @@ export const extension = { return configContainer; }, + /** + * Where each node stores files, keyed by node id. See `IConfig.servers`. A + * getter over live config rather than a copy, so it follows the same + * lazy-population rule as `config` above. Empty when none is configured. + */ + get servers(): NonNullable { + return configContainer.servers ?? {}; + }, + // -- Event subscription ------------------------------------------- on:

( diff --git a/src/backend/services/fs/FSService.test.ts b/src/backend/services/fs/FSService.test.ts index f346ffb6e..69c55e411 100644 --- a/src/backend/services/fs/FSService.test.ts +++ b/src/backend/services/fs/FSService.test.ts @@ -2202,18 +2202,21 @@ describe('FSService mkdir, touch, rename and shortcuts', () => { ); await writeFile(user, `${user.home}/Documents/taken.txt`, 'x'); - expect((await caught(() => fs.rename(user.userId, entry, 'a/b'))).message).toBe( - 'Name cannot contain a slash', - ); - expect((await caught(() => fs.rename(user.userId, entry, ' '))).message).toBe( - 'Name cannot be empty', - ); expect( - (await caught(() => fs.rename(user.userId, entry, 'taken.txt'))).statusCode, + (await caught(() => fs.rename(user.userId, entry, 'a/b'))).message, + ).toBe('Name cannot contain a slash'); + expect( + (await caught(() => fs.rename(user.userId, entry, ' '))).message, + ).toBe('Name cannot be empty'); + expect( + (await caught(() => fs.rename(user.userId, entry, 'taken.txt'))) + .statusCode, ).toBe(409); // Renaming to the current name is a no-op that returns the same row. - await expect(fs.rename(user.userId, entry, 'ren.txt')).resolves.toBe(entry); + await expect(fs.rename(user.userId, entry, 'ren.txt')).resolves.toBe( + entry, + ); }); it("refuses to rename another user's entry without write on it", async () => { @@ -3030,7 +3033,11 @@ describe('FSService restructuring a shared tree', () => { }); it('refuses rename to a recipient who holds only read', async () => { - const file = await writeFile(owner, `${owner.home}/lookdonttouch.txt`, 'x'); + const file = await writeFile( + owner, + `${owner.home}/lookdonttouch.txt`, + 'x', + ); await server.services.acl.setUserUser( owner.actor, holder.actor, @@ -3186,7 +3193,6 @@ describe('FSService ownership in a shared tree', () => { ); expect(inside?.userId).toBe(owner.userId); }); - }); describe('FSService storage allowance in a shared tree', () => { @@ -3219,7 +3225,9 @@ describe('FSService storage allowance in a shared tree', () => { limitedServer.stores.user, created, ); - const refreshed = (await limitedServer.stores.user.getById(created.id))!; + const refreshed = (await limitedServer.stores.user.getById( + created.id, + ))!; return { userId: refreshed.id, home: `/${username}`, @@ -3246,8 +3254,7 @@ describe('FSService storage allowance in a shared tree', () => { holder.actor, { path: shared.path, - resolveAncestors: () => - limitedFs.getAncestorChain(shared.path), + resolveAncestors: () => limitedFs.getAncestorChain(shared.path), }, 'write', ); @@ -3643,7 +3650,9 @@ describe('FSService — cross-app AppData access', () => { asCalendar(() => fs.remove(owner.userId, { entry: contactsFile })), ).rejects.toMatchObject({ statusCode: 403 }); await expect( - asCalendar(() => fs.rename(owner.userId, contactsFile, 'renamed.json')), + asCalendar(() => + fs.rename(owner.userId, contactsFile, 'renamed.json'), + ), ).rejects.toMatchObject({ statusCode: 403 }); const desktop = (await server.stores.fsEntry.getEntryByPath( @@ -3732,3 +3741,138 @@ describe('FSService — cross-app AppData access', () => { ).toBeFalsy(); }); }); + +describe('FSService home-region placement', () => { + let regionServer: PuterServer; + let regionFs: FSService; + + // A deployment that knows about two nodes. `node-far` is the home region + // under test; `node-here` stands in for this node's own bucket, which is + // configured separately below so the two are distinguishable. + beforeAll(async () => { + regionServer = await setupTestServer({ + s3_bucket: 'puter-local', + s3_region: 'us-west-2', + servers: { + 'node-far': { + bucket: 'puter-far', + bucketRegion: 'eu-central-1', + }, + 'node-here': { + bucket: 'puter-local', + bucketRegion: 'us-west-2', + }, + }, + } as never); + regionFs = regionServer.services.fs as unknown as FSService; + await regionServer.clients.s3 + .get('eu-central-1') + .send(new CreateBucketCommand({ Bucket: 'puter-far' })); + }); + + afterAll(async () => { + await regionServer?.shutdown(); + }); + + const regionUser = async () => { + const username = `fsr-${Math.random().toString(36).slice(2, 10)}`; + const created = await regionServer.stores.user.create({ + username, + uuid: uuidv4(), + password: null, + email: `${username}@test.local`, + free_storage: 100 * 1024 * 1024, + requires_email_confirmation: false, + }); + await generateDefaultFsentries( + regionServer.clients.db, + regionServer.stores.user, + created, + ); + return { userId: created.id, home: `/${username}` }; + }; + + const writeIn = async ( + user: { userId: number; home: string }, + name: string, + body: string, + homeRegion?: string, + extra: Record = {}, + ) => { + const { fsEntry } = await regionFs.write( + user.userId, + { + fileMetadata: { + path: `${user.home}/Documents/${name}`, + size: Buffer.byteLength(body), + contentType: 'text/plain', + ...extra, + }, + fileContent: body, + }, + undefined, + undefined, + homeRegion, + ); + return fsEntry; + }; + + const readFrom = async (entry: FSEntry): Promise => { + const result = await regionFs.readContent(entry, {}); + const chunks: Buffer[] = []; + for await (const chunk of result.body) chunks.push(Buffer.from(chunk)); + return Buffer.concat(chunks).toString(); + }; + + it("stores a write in the home region's bucket and reads it back", async () => { + const user = await regionUser(); + const entry = await writeIn(user, 'mail.eml', 'hello', 'node-far'); + + expect(entry.bucket).toBe('puter-far'); + expect(entry.bucketRegion).toBe('eu-central-1'); + // The row is what reads resolve from, so a foreign bucket has to be + // readable through the ordinary path. + expect(await readFrom(entry)).toBe('hello'); + }); + + it('falls back to this server’s bucket for a region it does not know', async () => { + const user = await regionUser(); + // A node id this deployment's config says nothing about. + const entry = await writeIn(user, 'unmapped.eml', 'hi', 'node-absent'); + + expect(entry.bucket).toBe('puter-local'); + expect(entry.bucketRegion).toBe('us-west-2'); + expect(await readFrom(entry)).toBe('hi'); + }); + + it('places a write with no home region exactly as before', async () => { + const user = await regionUser(); + const entry = await writeIn(user, 'plain.txt', 'unchanged'); + + expect(entry.bucket).toBe('puter-local'); + expect(entry.bucketRegion).toBe('us-west-2'); + }); + + it('keeps an overwrite where the row already points, home region or not', async () => { + const user = await regionUser(); + const first = await writeIn(user, 'pinned.eml', 'first', 'node-far'); + expect(first.bucket).toBe('puter-far'); + + // The entry outranks the hint: moving the bytes would repoint the row + // and strand the original object. + const second = await writeIn( + user, + 'pinned.eml', + 'second', + 'node-here', + { + overwrite: true, + }, + ); + + expect(second.uuid).toBe(first.uuid); + expect(second.bucket).toBe('puter-far'); + expect(second.bucketRegion).toBe('eu-central-1'); + expect(await readFrom(second)).toBe('second'); + }); +}); diff --git a/src/backend/services/fs/FSService.ts b/src/backend/services/fs/FSService.ts index c10dbe18b..4a8ec19d8 100644 --- a/src/backend/services/fs/FSService.ts +++ b/src/backend/services/fs/FSService.ts @@ -460,9 +460,37 @@ export class FSService extends PuterService { return bucketRegion; } + /** + * Where a write's bytes should land. A home region this deployment knows + * about names its own bucket; anything else — an unmapped region, a + * single-node deployment, no hint at all — falls back to this server's own + * bucket, because placement is an optimisation and storing the bytes is + * not. + */ + #resolvePlacement(homeRegion?: string): { + bucket: string; + bucketRegion: string; + } { + const place = homeRegion + ? this.config.servers?.[homeRegion] + : undefined; + if (homeRegion && !place) { + console.warn( + `[fs] no storage configured for region '${homeRegion}'; using this server's bucket`, + ); + } + return ( + place ?? { + bucket: this.#resolveBucket(), + bucketRegion: this.#resolveBucketRegion(), + } + ); + } + #normalizeWriteInput( userId: number, metadata: FSEntryWriteInput, + homeRegion?: string, ): NormalizedWriteInput { const normalizedPath = this.#normalizePath(metadata.path); if (normalizedPath === '/') { @@ -498,8 +526,7 @@ export class FSService extends PuterService { immutable: Boolean(metadata.immutable), isPublic: metadata.isPublic, multipartPartSize: metadata.multipartPartSize, - bucket: this.#resolveBucket(), - bucketRegion: this.#resolveBucketRegion(), + ...this.#resolvePlacement(homeRegion), }; } @@ -2798,10 +2825,15 @@ export class FSService extends PuterService { writeRequest: WriteRequest, uploadTracker?: UploadProgressTrackerLike, storageAllowanceMax?: number, + homeRegion?: string, ): Promise { + // Server-only, and a positional argument for that reason: the + // controllers build `writeRequest` out of the parsed request body, so a + // field there would let a caller choose its own bucket. let normalizedInput = this.#normalizeWriteInput( userId, writeRequest.fileMetadata, + homeRegion, ); const [resolvedTarget] = await this.#resolveWriteTargets(userId, [ { diff --git a/src/backend/services/fs/homeRegion.test.ts b/src/backend/services/fs/homeRegion.test.ts new file mode 100644 index 000000000..c9f8159d9 --- /dev/null +++ b/src/backend/services/fs/homeRegion.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest'; +import { DEFAULT_HOME_REGION, resolveHomeRegion } from './homeRegion.js'; + +describe('resolveHomeRegion', () => { + it("prefers the account's own home region", () => { + expect( + resolveHomeRegion({ home: 'node-a', signup_server: 'node-b' }), + ).toBe('node-a'); + }); + + it('falls back to the server that served the signup', () => { + // Every account that predates the `home` column is this case. + expect(resolveHomeRegion({ home: null, signup_server: 'node-b' })).toBe( + 'node-b', + ); + expect(resolveHomeRegion({ signup_server: 'node-c' })).toBe('node-c'); + }); + + it('falls back to the primary region when neither is set', () => { + expect(resolveHomeRegion({ home: null, signup_server: null })).toBe( + 'oregon', + ); + expect(resolveHomeRegion({})).toBe(DEFAULT_HOME_REGION); + }); + + it('treats an empty string as unset rather than as a region', () => { + expect(resolveHomeRegion({ home: '', signup_server: 'node-d' })).toBe( + 'node-d', + ); + expect(resolveHomeRegion({ home: '', signup_server: '' })).toBe( + 'oregon', + ); + }); +}); diff --git a/src/backend/services/fs/homeRegion.ts b/src/backend/services/fs/homeRegion.ts new file mode 100644 index 000000000..50c36f54f --- /dev/null +++ b/src/backend/services/fs/homeRegion.ts @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Which home region an account's data belongs in. + * + * A home region is a node id, not a storage region: `IConfig.servers` maps one + * to the bucket that holds it. The ids are the same ones `config.serverId` and + * `user.signup_server` already use, so no translation is needed between them. + */ + +/** Where an account's data belongs when nothing else says otherwise. */ +export const DEFAULT_HOME_REGION = 'oregon'; + +/** + * An account's own `home`, else the server that served its signup, else the + * deployment default. Empty strings count as unset: a blank column is not a + * region. + */ +export const resolveHomeRegion = (user: { + home?: string | null; + signup_server?: string | null; +}): string => user.home || user.signup_server || DEFAULT_HOME_REGION; diff --git a/src/backend/stores/user/UserStore.ts b/src/backend/stores/user/UserStore.ts index b4e8cb729..2fde969f8 100644 --- a/src/backend/stores/user/UserStore.ts +++ b/src/backend/stores/user/UserStore.ts @@ -73,6 +73,13 @@ export interface UserRow { * did. */ card_fingerprint?: string | null; + /** + * Home region this account's data belongs in - a node id from + * `IConfig.servers`, not a storage region. Null for accounts that predate + * the column, which fall back to `signup_server`; `resolveHomeRegion` is + * the one place that precedence lives. + */ + home?: string | null; password?: string; [k: string]: unknown; } diff --git a/src/backend/types.ts b/src/backend/types.ts index 3ffefe103..fc334491d 100644 --- a/src/backend/types.ts +++ b/src/backend/types.ts @@ -928,6 +928,12 @@ interface IConfigOptional { s3_region: string; /** Fallback AWS region. */ region: string; + /** + * Where each node stores files, keyed by node id - the same ids `user.home` + * names. A write may pass one to place its bytes there; absent, or absent + * an entry, every write uses `s3_bucket` / `s3_region`. + */ + servers?: Record; /** Default storage capacity per user (bytes). */ storage_capacity: number; /** When false, storage is effectively unlimited (bounded by device space). */