From 1fc2eae1ae3a8312468f2ca0a3a7fab274c8f982 Mon Sep 17 00:00:00 2001 From: Neal Shah <30693865+ProgrammerIn-wonderland@users.noreply.github.com> Date: Fri, 25 Sep 2026 21:48:15 -0400 Subject: [PATCH] self hosted mail (#3932) * self hosted mail * update lock * address daniel review --- .env.example | 5 + Dockerfile | 2 + config.template.jsonc | 32 +++ doc/self-hosting.md | 95 +++++++ docker-compose.yml | 26 ++ extensions/useremailrx/index.test.ts | 268 ++++++++++++++++++++ extensions/useremailrx/index.ts | 104 ++++++++ extensions/useremailrx/package-lock.json | 12 + extensions/useremailrx/package.json | 6 + package-lock.json | 53 ++++ package.json | 1 + src/backend/config.ts | 159 ++++++++++++ src/backend/index.ts | 141 +---------- src/backend/package.json | 3 + src/backend/services/email/mailbox.test.ts | 223 +++++++++++++++++ src/backend/services/email/mailbox.ts | 186 ++++++++++++++ src/backend/smtp/SmtpReceiver.test.ts | 274 +++++++++++++++++++++ src/backend/smtp/SmtpReceiver.ts | 168 +++++++++++++ src/backend/smtp/config.test.ts | 155 ++++++++++++ src/backend/smtp/config.ts | 119 +++++++++ src/backend/smtp/index.ts | 72 ++++++ src/backend/smtp/ingressClient.test.ts | 205 +++++++++++++++ src/backend/smtp/ingressClient.ts | 160 ++++++++++++ src/backend/smtp/recipients.test.ts | 65 +++++ src/backend/smtp/recipients.ts | 51 ++++ src/backend/types.ts | 42 ++++ tools/start.mjs | 18 +- 27 files changed, 2504 insertions(+), 141 deletions(-) create mode 100644 extensions/useremailrx/index.test.ts create mode 100644 extensions/useremailrx/index.ts create mode 100644 extensions/useremailrx/package-lock.json create mode 100644 extensions/useremailrx/package.json create mode 100644 src/backend/config.ts create mode 100644 src/backend/services/email/mailbox.test.ts create mode 100644 src/backend/services/email/mailbox.ts create mode 100644 src/backend/smtp/SmtpReceiver.test.ts create mode 100644 src/backend/smtp/SmtpReceiver.ts create mode 100644 src/backend/smtp/config.test.ts create mode 100644 src/backend/smtp/config.ts create mode 100644 src/backend/smtp/index.ts create mode 100644 src/backend/smtp/ingressClient.test.ts create mode 100644 src/backend/smtp/ingressClient.ts create mode 100644 src/backend/smtp/recipients.test.ts create mode 100644 src/backend/smtp/recipients.ts diff --git a/.env.example b/.env.example index 726f74601..21f996ee1 100644 --- a/.env.example +++ b/.env.example @@ -16,3 +16,8 @@ MARIADB_PASSWORD=replace-with-strong-password S3_ACCESS_KEY=puter S3_SECRET_KEY=replace-with-strong-secret S3_BUCKET=puter-local + +# ── Inbound SMTP (compose profile `smtp`) ------------------------------ +# Host port mapped to the receiver's 2525. 25 is what an MX record means; +# many hosting providers block it. +SMTP_PORT=25 diff --git a/Dockerfile b/Dockerfile index 15bb3d133..3f612de65 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,6 +79,8 @@ ENV PUTER_CONFIG_PATH=/etc/puter/config.json ENV NODE_OPTIONS=--enable-source-maps EXPOSE 4100 +# The inbound SMTP receiver, when run from this image with the `smtp` profile. +EXPOSE 2525 USER node diff --git a/config.template.jsonc b/config.template.jsonc index 66123edfa..7df1c8d84 100644 --- a/config.template.jsonc +++ b/config.template.jsonc @@ -255,6 +255,38 @@ } }, + // ── Inbound mail (separate process) ───────────────────────────────── + // Mailboxes addressed at one of your own domains. `secret` gates the + // ingress endpoint that files a message into the recipient's ~/.mail. + // + // `localServer` additionally runs an SMTP listener that accepts mail from + // other mail servers and hands each message to that endpoint. It runs as + // its own process - `npm run start:smtp`, or the `smtp` compose profile - + // and stores nothing itself. Without it, nothing delivers to the endpoint + // unless you point something else at it. + // + // The listener is unauthenticated by nature: anything on the internet may + // hand it a message for an address in `localDomains`. It performs no spam, + // SPF, DKIM or DMARC checks. Read the self-hosting guide before exposing + // it. + // "userEmail": { + // "secret": "a long random string", + // "localServer": true, + // "localDomains": ["example.com"], + // // Container-internal by default; publish 25 onto it from outside. + // "localPort": 2525, + // // Defaults to /email/ingress. The endpoint is served + // // on the `api` subdomain, so when this points somewhere internal, + // // `localIngressHost` has to name that virtual host. + // "localIngressUrl": "http://puter:4100/email/ingress", + // "localIngressHost": "api.example.com", + // "localHostname": "mx.example.com", + // "localMaxRecipients": 50, + // // Concurrent connections. Each message is held in memory while it + // // is delivered, so this also bounds memory use. + // "localMaxClients": 20 + // }, + // ── Sharing ───────────────────────────────────────────────────────── // Email a recipient who already has an account about a new share. On // unless set to false; they opt out via the unsubscribe link, or by diff --git a/doc/self-hosting.md b/doc/self-hosting.md index 7c8b897f1..d93a8a385 100644 --- a/doc/self-hosting.md +++ b/doc/self-hosting.md @@ -35,6 +35,12 @@ Optional services (compose profile `ai`, opt-in): | `puter-ollama` | `ollama/ollama` | Local LLM provider (CPU; GPU passthrough opt-in) | | `puter-ollama-init` | `ollama/ollama` | One-shot — pulls the default model (`tinyllama`) on first boot | +Optional services (compose profile `smtp`, opt-in): + +| Container | Image | Role | +| ------------ | ------------------------ | -------------------------------------------------------- | +| `puter-smtp` | `ghcr.io/heyputer/puter` | Receives mail over SMTP and files it into user mailboxes | + State lives under `./puter/data//`. --- @@ -513,6 +519,95 @@ Without `--profile ai`, the `ollama` containers stay down and Puter (with `enabl For GPU acceleration (NVIDIA), uncomment the `deploy:` block under the `ollama` service in [docker-compose.yml](../docker-compose.yml). Requires `nvidia-container-toolkit` on the host. +## Optional: receiving mail over SMTP + +Puter can give each account a mailbox at a domain you control. Two pieces are +involved, and they are configured together under `userEmail`: + +- an **ingress endpoint** (`POST /email/ingress`) that files a message into the + recipient's `~/.mail`. It exists whenever `userEmail.secret` is set. +- the **`puter-smtp` receiver**, which accepts mail from other mail servers and + hands each message to that endpoint. It runs as its own process, behind the + `smtp` compose profile. + +> **Read this before you expose it.** An inbound mail server is unauthenticated +> by nature: anyone on the internet may hand it a message for any address in +> `localDomains`. Puter performs **no** spam filtering and **no** SPF, DKIM or +> DMARC checks — whatever is delivered is stored in the recipient's mailbox and +> counts against their storage. If that is not acceptable, put a filtering relay +> in front of it and point `localDomains` at that instead. + +1. Configure it in `puter/config/config.json`: + + ```json + "userEmail": { + "secret": "a long random string", + "localServer": true, + "localDomains": ["example.com"] + } + ``` + + `secret` is shared between the two halves; generate it with + `openssl rand -hex 32`. `localDomains` lists the domains this server accepts + mail for — anything else is refused at `RCPT TO`, which is what stops it + being used as an open relay. It has no default, and the receiver refuses to + start without it. + + The receiver posts to `/email/ingress` unless you set + `localIngressUrl`. The endpoint is served on the `api` subdomain, so if you + point it at an internal address to skip the reverse proxy, name that virtual + host too or the request will 404: + + ```json + "localIngressUrl": "http://puter:4100/email/ingress", + "localIngressHost": "api.example.com" + ``` + + `localIngressHost` defaults to the host of `api_base_url`, which is usually + what you want. + +2. Bring it up with the `smtp` profile: + + ```bash + docker compose --profile smtp up -d + docker compose logs -f puter-smtp + ``` + +3. Point DNS at it: + + ```dns + mx.example.com. A 203.0.113.10 + example.com. MX 10 mx.example.com. + ``` + + Also set a `PTR` record for the address that resolves back to + `mx.example.com` — many senders refuse mail from a server without one. + + Inbound TCP port 25 must reach the host. **Most cloud providers block port + 25 by default and residential ISPs block it permanently**; if you cannot get + it unblocked, this will not work no matter how it is configured. The + container itself binds 2525 (it runs unprivileged) and compose publishes + `${SMTP_PORT:-25}` onto it, so you can move the host port if you need to. + +A message is held in memory while it is being delivered, so peak memory is +bounded by `localMaxClients` (default 20) times the 25 MiB message ceiling. +Lower `localMaxClients` on a small host. There is no queue and no retry +schedule — if delivery fails, the sending server is told to try again later and +retrying is its job. + +Mail is stored as `message/rfc822` objects under `~/.mail/objects//` and +read back through `puter.email` in the SDK. + +Every setting is documented in +[config.template.jsonc](../config.template.jsonc). Two deliberate limits, both +in the name of keeping the self-hosted path simple: it does not offer STARTTLS, +so mail arrives in the clear, and it applies no per-sender rate limiting. Put a +filtering relay in front if either matters to you. + +Note this is a *receiving* server only: it never sends, and it must not be used +for authenticated submission. Outbound transactional mail is the separate +`email` block above. + ## Building from source instead of pulling To run a local build against the full stack, use a source checkout and complete diff --git a/docker-compose.yml b/docker-compose.yml index ffba46bee..7dc2c622b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -286,6 +286,32 @@ services: retries: 3 start_period: 30s + # Inbound mail. Behind the `smtp` profile, so it only starts when asked for: + # docker compose --profile smtp up -d + # Accepts SMTP from other mail servers and hands each message to the ingress + # endpoint, which files it into the recipient's mailbox. Needs `userEmail` + # configured — see doc/self-hosting.md#optional-receiving-mail-over-smtp. + puter-smtp: + profiles: ["smtp"] + image: ghcr.io/heyputer/puter:main + pull_policy: always + container_name: puter-smtp + restart: unless-stopped + command: ["node", "-r", "./dist/src/backend/telemetry.js", "./dist/src/backend/smtp/index.js"] + depends_on: + puter: + condition: service_healthy + ports: + # The container runs unprivileged and binds 2525; 25 is what an MX record + # means, and many providers block it. + - "${SMTP_PORT:-25}:2525" + environment: + PUID: 1000 + PGID: 1000 + volumes: + # The same config.json the backend reads; only `userEmail` applies here. + - ./puter/config:/etc/puter:z + caddy: image: caddy:2.11-alpine container_name: puter-caddy diff --git a/extensions/useremailrx/index.test.ts b/extensions/useremailrx/index.test.ts new file mode 100644 index 000000000..82954822c --- /dev/null +++ b/extensions/useremailrx/index.test.ts @@ -0,0 +1,268 @@ +import { EventEmitter } from 'node:events'; +import { Readable } from 'node:stream'; +import type { Request, Response } from 'express'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const SECRET = 'ingress-secret'; + +// Only `extension` is stubbed; the secret comparison and the routing are the +// real code paths, and the fake service/store layer is the boundary. +const state = vi.hoisted(() => ({ + config: { userEmail: { secret: 'ingress-secret' } } as Record< + string, + unknown + >, + routes: [] as Array<{ + path: string; + handler: (req: unknown, res: unknown) => unknown; + }>, + users: new Map>(), + writes: [] as Array<{ + userId: number; + path: string; + homeRegion?: string; + }>, +})); + +vi.mock('@heyputer/backend/src/extensions', () => ({ + extension: { + get config() { + return state.config; + }, + post( + path: string, + _opts: unknown, + handler: (req: unknown, res: unknown) => unknown, + ) { + state.routes.push({ path, handler }); + }, + import(layer: string) { + if (layer === 'store') { + return { + user: { + getByUsername: async (username: string) => + state.users.get(username) ?? null, + }, + }; + } + return { + fs: { + async write( + userId: number, + { fileMetadata }: { fileMetadata: { path: string } }, + _uploadTracker?: unknown, + _storageAllowanceMax?: number, + homeRegion?: string, + ) { + state.writes.push({ + userId, + path: fileMetadata.path, + homeRegion, + }); + return { fsEntry: { path: fileMetadata.path } }; + }, + }, + }; + }, + }, +})); + +import './index.js'; + +const permanent = (id: number, username: string) => ({ + id, + username, + email: `${username}@example.com`, + password: '$2b$10$hash', +}); +/** No password and no email is what makes an account temporary. */ +const temp = (id: number, username: string) => ({ + id, + username, + email: null, + password: null, +}); + +const ingress = async ( + to: string, + over: { secret?: string; contentLength?: string | null } = {}, +) => { + const route = state.routes.find((r) => r.path === '/email/ingress'); + if (!route) throw new Error('route not registered'); + + const body = Buffer.from('From: someone@example.com\r\n\r\nhello\r\n'); + const req = Readable.from([body], { objectMode: false }) as unknown as { + headers: Record; + query: Record; + }; + // `contentLength: null` sends no length header at all, which is the one + // framing the HTTP parser does not bound for us. + req.headers = + over.contentLength === null + ? {} + : { + 'content-length': over.contentLength ?? String(body.length), + }; + req.query = { SECRET: over.secret ?? SECRET, to, subject: 'hi' }; + + let status = 200; + let ended = false; + // `end` takes (chunk, encoding) and no callback, the way the compression + // middleware leaves it, and emits `finish` — which is what the refusal + // paths hang their drain/destroy on. A stub that accepts a callback + // instead hides the bug that took the 413 path to production as a 500. + const emitter = new EventEmitter(); + const res = { + status(code: number) { + status = code; + return res; + }, + end(chunk?: unknown, encoding?: BufferEncoding) { + if (chunk) Buffer.byteLength(chunk as string, encoding); + ended = true; + emitter.emit('finish'); + }, + once(event: string, listener: () => void) { + emitter.once(event, listener); + return res; + }, + } as unknown as Response; + + await route.handler(req as unknown as Request, res); + return { + status, + ended, + drained: (req as unknown as Readable).readableFlowing, + destroyed: (req as unknown as Readable).destroyed, + }; +}; + +beforeEach(() => { + state.users.clear(); + state.writes.length = 0; +}); + +describe('temporary accounts cannot receive', () => { + test('mail for a temp account is accepted and dropped', async () => { + state.users.set('tempy', temp(1, 'tempy')); + + const { status, ended, drained } = await ingress('tempy@puter.email'); + // Accepted, so the relay neither retries nor bounces to the sender. + expect(status).toBe(200); + expect(ended).toBe(true); + // The body is read off the wire rather than left unconsumed. + expect(drained).toBe(true); + expect(state.writes).toEqual([]); + }); + + test('mail for a permanent account still lands in the mailbox', async () => { + state.users.set('bob', permanent(2, 'bob')); + + const { status } = await ingress('bob@puter.email'); + expect(status).toBe(200); + expect(state.writes).toHaveLength(1); + expect(state.writes[0].userId).toBe(2); + expect(state.writes[0].path).toMatch(/^\/bob\/\.mail\/objects\//); + }); + + test('an unknown address is still rejected outright', async () => { + const { status } = await ingress('nobody@puter.email'); + expect(status).toBe(404); + expect(state.writes).toEqual([]); + }); +}); + +describe('inbound mail is placed in the recipient home region', () => { + /** A permanent account with explicit placement columns. */ + const placed = ( + id: number, + username: string, + over: { home?: string | null; signup_server?: string | null }, + ) => ({ + ...permanent(id, username), + home: null, + signup_server: null, + ...over, + }); + + test("uses the recipient's own home region when they have one", async () => { + state.users.set('bob', placed(2, 'bob', { home: 'frankfurt' })); + + await ingress('bob@puter.email'); + expect(state.writes[0].homeRegion).toBe('frankfurt'); + }); + + test('falls back to the server that served their signup', async () => { + // Every account that predates the `home` column is this case. + state.users.set( + 'bob', + placed(2, 'bob', { home: null, signup_server: 'london' }), + ); + + await ingress('bob@puter.email'); + expect(state.writes[0].homeRegion).toBe('london'); + }); + + test('falls back to the primary region when neither is set', async () => { + state.users.set('bob', placed(2, 'bob', {})); + + await ingress('bob@puter.email'); + expect(state.writes[0].homeRegion).toBe('oregon'); + }); + + test('home wins over signup_server when both are set', async () => { + state.users.set( + 'bob', + placed(2, 'bob', { home: 'sydney', signup_server: 'london' }), + ); + + await ingress('bob@puter.email'); + expect(state.writes[0].homeRegion).toBe('sydney'); + }); +}); + +describe('refusals dispose of the body nobody read', () => { + test('an unknown recipient drains it and keeps the connection', async () => { + // The secret checked out and the length is under the cap, so the body + // was ours to read: discard it rather than resetting a usable socket. + const { status, drained, destroyed } = + await ingress('nobody@puter.email'); + expect(status).toBe(404); + expect(drained).toBe(true); + expect(destroyed).toBe(false); + }); + + test('a bad secret closes rather than reading the upload', async () => { + state.users.set('bob', permanent(2, 'bob')); + + const { status, destroyed } = await ingress('bob@puter.email', { + secret: 'not-the-secret', + }); + expect(status).toBe(403); + // Nothing authenticated this caller; their bytes are not worth reading. + expect(destroyed).toBe(true); + expect(state.writes).toEqual([]); + }); + + test('no content-length is refused with 411', async () => { + state.users.set('bob', permanent(2, 'bob')); + + const { status, destroyed } = await ingress('bob@puter.email', { + contentLength: null, + }); + expect(status).toBe(411); + expect(destroyed).toBe(true); + expect(state.writes).toEqual([]); + }); + + test('a declared length over the cap is refused with 413', async () => { + state.users.set('bob', permanent(2, 'bob')); + + const { status, destroyed } = await ingress('bob@puter.email', { + contentLength: String(25 * 1024 * 1024 + 1), + }); + expect(status).toBe(413); + expect(destroyed).toBe(true); + expect(state.writes).toEqual([]); + }); +}); diff --git a/extensions/useremailrx/index.ts b/extensions/useremailrx/index.ts new file mode 100644 index 000000000..5aa0cbfae --- /dev/null +++ b/extensions/useremailrx/index.ts @@ -0,0 +1,104 @@ +import { extension } from '@heyputer/backend/src/extensions'; +import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto'; +import { + MAX_MESSAGE_BYTES, + isTempUser, + puterEmailUsername, + storeInboxMessage, +} from '@heyputer/backend/src/services/email/mailbox.js'; + +// copied form peer secrets checker +const COMPARE_KEY = randomBytes(32); +const secretsEqual = (a: string, b: string): boolean => + timingSafeEqual( + createHmac('sha256', COMPARE_KEY).update(a).digest(), + createHmac('sha256', COMPARE_KEY).update(b).digest(), + ); + +const INGRESS_SECRET = ( + (extension.config as Record).userEmail as + | { secret?: string } + | undefined +)?.secret; + +if (!INGRESS_SECRET) { + console.warn( + '[useremailrx] config.userEmail.secret unset - email ingress disabled', + ); +} else { + extension.post( + '/email/ingress', + { + subdomain: 'api', + }, + async (req, res) => { + // Answer first, then deal with the body nobody read. Touching the + // request before the response has flushed takes the response down + // with the connection and the caller sees a reset with no status, + // so this hangs off `finish` — and it has to be the event rather + // than `res.end(cb)`, because the compression middleware replaces + // `res.end` with an `(chunk, encoding)` signature that has no + // callback parameter, so a callback lands in `chunk` and dies in + // `Buffer.byteLength`. + // + // `drain` discards the rest of a body we were entitled to read and + // leaves the connection reusable. `close` is for bodies we do not + // want to read at all: an unauthenticated caller's, or one whose + // declared length is missing or over the cap. + const refuse = (status: number, how: 'drain' | 'close') => { + res.once('finish', () => + how === 'drain' ? req.resume() : req.destroy(), + ); + res.status(status).end(); + }; + + const presented = req.query.SECRET; + if ( + typeof presented !== 'string' || + !secretsEqual(presented, INGRESS_SECRET) + ) { + // Nothing has authenticated this caller, so refuse to spend + // bandwidth reading whatever they were uploading. + refuse(403, 'close'); + return; + } + + const size = Number(req.headers['content-length']); + if (!Number.isInteger(size) || size <= 0) { + refuse(411, 'close'); + return; + } + if (size > MAX_MESSAGE_BYTES) { + refuse(413, 'close'); + return; + } + + const stores = extension.import('store'); + const services = extension.import('service'); + + const user = await stores.user.getByUsername( + puterEmailUsername(req.query.to as string), + ); + if (!user) { + refuse(404, 'drain'); + return; + } + + if (isTempUser(user)) { + req.resume(); + res.end(); + return; + } + + await storeInboxMessage(services.fs, user, { + subject: req.query.subject, + content: req, + size, + }); + + const { SECRET: _secret, ..._loggableQuery } = req.query; + // console.log('got email: ', puterUser, loggableQuery, size); + res.end(); + }, + ); +} diff --git a/extensions/useremailrx/package-lock.json b/extensions/useremailrx/package-lock.json new file mode 100644 index 000000000..bab0e48a9 --- /dev/null +++ b/extensions/useremailrx/package-lock.json @@ -0,0 +1,12 @@ +{ + "name": "emailrx", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "emailrx", + "version": "1.0.0" + } + } +} diff --git a/extensions/useremailrx/package.json b/extensions/useremailrx/package.json new file mode 100644 index 000000000..56ae6f0c2 --- /dev/null +++ b/extensions/useremailrx/package.json @@ -0,0 +1,6 @@ +{ + "name": "emailrx", + "version": "1.0.0", + "main": "index.ts", + "description": "Internal service which handles routing puter emails to users" +} diff --git a/package-lock.json b/package-lock.json index e54a1b80b..87b4a9fb2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1216,6 +1216,7 @@ }, "node_modules/@clack/prompts/node_modules/is-unicode-supported": { "version": "1.3.0", + "extraneous": true, "inBundle": true, "license": "MIT", "engines": { @@ -6153,6 +6154,17 @@ "@types/node": "*" } }, + "node_modules/@types/smtp-server": { + "version": "3.5.13", + "resolved": "https://registry.npmjs.org/@types/smtp-server/-/smtp-server-3.5.13.tgz", + "integrity": "sha512-S3rGl2KbViH+98/CgHipPIWgtAFnjxLsppxIRbgJHNuZL7Y+py+7kZjT7xS+wOmCxYTn4O7IqLqkvekg99MDVQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/nodemailer": "*" + } + }, "node_modules/@types/tedious": { "version": "4.0.14", "resolved": "https://registry.npmjs.org/@types/tedious/-/tedious-4.0.14.tgz", @@ -10797,6 +10809,12 @@ "node": ">= 10" } }, + "node_modules/ipv6-normalize": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ipv6-normalize/-/ipv6-normalize-1.0.1.tgz", + "integrity": "sha512-Bm6H79i01DjgGTCWjUuCjJ6QDo1HB96PT/xCYuyJUP9WFbVDrLSbG4EZCvOCun2rNswZb0c3e4Jt/ws795esHA==", + "license": "MIT" + }, "node_modules/is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -14488,6 +14506,15 @@ "node": ">=6" } }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/puter-mcp-connector": { "resolved": "src/mcp-connector", "link": true @@ -15530,6 +15557,29 @@ "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", "license": "MIT" }, + "node_modules/smtp-server": { + "version": "3.19.13", + "resolved": "https://registry.npmjs.org/smtp-server/-/smtp-server-3.19.13.tgz", + "integrity": "sha512-W1CjNcrPqW4+1gUOYRyjPvTOxu7GgKR9+7l50yQBF0GM8v8jpYwheHFCHX7s+p7pYI8iXKQQZiF2Rv2Na/3i2g==", + "license": "MIT-0", + "dependencies": { + "ipv6-normalize": "1.0.1", + "nodemailer": "10.0.10", + "punycode.js": "2.3.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/smtp-server/node_modules/nodemailer": { + "version": "10.0.10", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-10.0.10.tgz", + "integrity": "sha512-He9XskOFms62SyAKkLu8CcGcYHAo+BSHSsORI8s4PJH8qZgZY4L4lDfvyN0twvmbpyG+eGrxpyBlskj9d9rJ8A==", + "license": "MIT-0", + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/socket.io": { "version": "4.8.3", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.8.3.tgz", @@ -17650,8 +17700,10 @@ "openai": "^6.34.0", "otpauth": "^9.5.2", "pg": "^8.21.0", + "postal-mime": "^3.0.0", "replicate": "^1.0.0", "sharp": "^0.35.4", + "smtp-server": "^3.19.13", "socket.io": "^4.8.3", "svg-captcha": "^1.4.0", "together-ai": "^0.33.0", @@ -17670,6 +17722,7 @@ "@types/node": "^24.13.6", "@types/nodemailer": "^8.0.1", "@types/pg": "^8.6.1", + "@types/smtp-server": "^3.5.13", "@types/validator": "^13.15.10", "pgmock": "^1.0.3", "typescript": "^5.9.3", diff --git a/package.json b/package.json index d2003adab..a9b2c109d 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "build:workerLib:coverage": "cd src/puter-js && npm run build:coverage && cd ../worker && npm run build", "start:gui": "nodemon --exec \"node dev-server.js\" ", "start": "node ./tools/start.mjs", + "start:smtp": "node ./tools/start.mjs --smtp", "dev": "npm start", "build": "npm run setupExtensions && npm run build:ts && cd src/gui && node ./build.js && cd ../puter-js && npm run build", "build:workerLib": "cd src/puter-js && npm run build && cd ../worker && npm run build", diff --git a/src/backend/config.ts b/src/backend/config.ts new file mode 100644 index 000000000..0bd702e2e --- /dev/null +++ b/src/backend/config.ts @@ -0,0 +1,159 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import type { IConfig } from './types'; + +// Config resolution order: +// 1. `process.env.PUTER_CONFIG_PATH` — absolute path to a config file. Used +// by deployments whose outer bootstrap writes a merged config to a known +// location. +// 2. `/config.json` — user's runtime override (gitignored), +// deep-merged over config.default.json so users can omit keys they +// don't care to override (e.g. gui_assets_root, database). +// 3. `/config.default.json` — bundled OSS defaults. +// +// Post-flatten depth: compiled file is at `packages/puter/dist/src/backend/index.js`, +// so three `..`s land at `packages/puter/`. +const PACKAGE_ROOT = path.resolve(__dirname, '../../..'); +// Root of the running code tree. Matches PACKAGE_ROOT for a source run, but +// points at `dist/` for a compiled run — so config-declared paths like +// `./extensions` resolve to `dist/extensions` at runtime without the config +// having to know about the build layout. +const RUNTIME_ROOT = path.resolve(__dirname, '../..'); + +const isPlainObject = (v: unknown): v is Record => + typeof v === 'object' && v !== null && !Array.isArray(v); + +const deepMerge = >( + base: T, + override: Record, +): T => { + const out: Record = { ...base }; + for (const [k, v] of Object.entries(override)) { + out[k] = + isPlainObject(v) && isPlainObject(out[k]) + ? deepMerge(out[k] as Record, v) + : v; + } + return out as T; +}; + +export const loadConfig = (): IConfig => { + const envPath = process.env.PUTER_CONFIG_PATH; + const runtimePath = path.join(PACKAGE_ROOT, 'config.json'); + const defaultPath = path.join(PACKAGE_ROOT, 'config.default.json'); + + const defaults = existsSync(defaultPath) + ? (JSON.parse(readFileSync(defaultPath, 'utf8')) as Record< + string, + unknown + >) + : {}; + + // Runtime override path: env wins, then config.json, else no override + // (we still return defaults so single-file installs work). + const overridePath = + envPath && existsSync(envPath) + ? envPath + : existsSync(runtimePath) + ? runtimePath + : null; + + console.log(`[config] defaults from ${defaultPath}`); + if (overridePath) console.log(`[config] override from ${overridePath}`); + + const override = overridePath + ? (JSON.parse(readFileSync(overridePath, 'utf8')) as Record< + string, + unknown + >) + : {}; + + const config = deepMerge(defaults, override) as IConfig; + + if (!config.version) { + const pkgPath = path.join(PACKAGE_ROOT, 'package.json'); + if (existsSync(pkgPath)) { + try { + const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')) as { + version?: string; + }; + if (pkg.version) config.version = pkg.version; + } catch { + // fall through — /version returns 'unknown' + } + } + } + + // Computed defaults. `origin` and `pub_port` are the externally-visible + // URL+port — what the browser sees. Separate from `port`, which is the + // bind port (can differ when behind a reverse proxy). Code paths that + // build self-referential URLs (GUI bootstrap, email links, OIDC callbacks) + // depend on `origin` having the right port baked in. + if (config.pub_port === undefined) config.pub_port = config.port; + const protocol = config.protocol ?? 'http'; + const domain = config.domain ?? 'localhost'; + const portSuffix = + config.pub_port === 80 || config.pub_port === 443 + ? '' + : `:${config.pub_port}`; + if (config.origin === undefined) { + config.origin = `${protocol}://${domain}${portSuffix}`; + } + // API lives on the `api.` subdomain on the same host+port as the main + // origin (see PuterRouter subdomain handling in server.ts). Compute it + // from pub_port/domain so a single-port override (e.g. port: 5101) flows + // through to the GUI bootstrap without users having to restate the URL. + if (config.api_base_url === undefined) { + config.api_base_url = `${protocol}://api.${domain}${portSuffix}`; + } + + // Resolve path-valued config fields. Two different roots: + // - `extensions` uses RUNTIME_ROOT because extensions ship inside the + // build output (dist/extensions) and the loader's dynamic import() + // resolves relative paths against the *importing* module file. + // - GUI/puter-js/builtin-apps use PACKAGE_ROOT because those assets + // live only in the source tree (not copied into dist/) and are served + // via express.static at runtime. + const resolveRuntime = (p: string): string => + path.isAbsolute(p) ? p : path.resolve(RUNTIME_ROOT, p); + const resolvePackage = (p: string): string => + path.isAbsolute(p) ? p : path.resolve(PACKAGE_ROOT, p); + + if (Array.isArray(config.extensions)) { + config.extensions = config.extensions.map(resolveRuntime); + } + if (typeof config.gui_assets_root === 'string') { + config.gui_assets_root = resolvePackage(config.gui_assets_root); + } + if (typeof config.puterjs_root === 'string') { + config.puterjs_root = resolvePackage(config.puterjs_root); + } + if (isPlainObject(config.builtin_apps)) { + for (const [k, v] of Object.entries(config.builtin_apps)) { + if (typeof v === 'string') { + (config.builtin_apps as Record)[k] = + resolvePackage(v); + } + } + } + return config; +}; diff --git a/src/backend/index.ts b/src/backend/index.ts index 983beb1e6..c5d7b6078 100644 --- a/src/backend/index.ts +++ b/src/backend/index.ts @@ -17,160 +17,21 @@ * along with this program. If not, see . */ -import { existsSync, readFileSync } from 'node:fs'; -import path from 'node:path'; import { isSpanContextValid, trace } from '@opentelemetry/api'; import { puterClients } from './clients'; +import { loadConfig } from './config'; import { puterControllers } from './controllers'; import { puterDrivers } from './drivers'; import { PuterServer } from './server'; import { puterServices } from './services'; import { puterStores } from './stores'; -import type { IConfig } from './types'; import { createGracefulShutdown } from './util/gracefulShutdown.js'; import { installJsonConsole } from './util/jsonConsole.js'; -// Config resolution order: -// 1. `process.env.PUTER_CONFIG_PATH` — absolute path to a config file. Used -// by deployments whose outer bootstrap writes a merged config to a known -// location. -// 2. `/config.json` — user's runtime override (gitignored), -// deep-merged over config.default.json so users can omit keys they -// don't care to override (e.g. gui_assets_root, database). -// 3. `/config.default.json` — bundled OSS defaults. -// -// Post-flatten depth: compiled file is at `packages/puter/dist/src/backend/index.js`, -// so three `..`s land at `packages/puter/`. -const PACKAGE_ROOT = path.resolve(__dirname, '../../..'); -// Root of the running code tree. Matches PACKAGE_ROOT for a source run, but -// points at `dist/` for a compiled run — so config-declared paths like -// `./extensions` resolve to `dist/extensions` at runtime without the config -// having to know about the build layout. -const RUNTIME_ROOT = path.resolve(__dirname, '../..'); - // How long a node with a server identity keeps serving in-flight work after // it stops accepting connections. const GRACEFUL_DRAIN_MS = 90_000; -const isPlainObject = (v: unknown): v is Record => - typeof v === 'object' && v !== null && !Array.isArray(v); - -const deepMerge = >( - base: T, - override: Record, -): T => { - const out: Record = { ...base }; - for (const [k, v] of Object.entries(override)) { - out[k] = - isPlainObject(v) && isPlainObject(out[k]) - ? deepMerge(out[k] as Record, v) - : v; - } - return out as T; -}; - -const loadConfig = (): IConfig => { - const envPath = process.env.PUTER_CONFIG_PATH; - const runtimePath = path.join(PACKAGE_ROOT, 'config.json'); - const defaultPath = path.join(PACKAGE_ROOT, 'config.default.json'); - - const defaults = existsSync(defaultPath) - ? (JSON.parse(readFileSync(defaultPath, 'utf8')) as Record< - string, - unknown - >) - : {}; - - // Runtime override path: env wins, then config.json, else no override - // (we still return defaults so single-file installs work). - const overridePath = - envPath && existsSync(envPath) - ? envPath - : existsSync(runtimePath) - ? runtimePath - : null; - - console.log(`[config] defaults from ${defaultPath}`); - if (overridePath) console.log(`[config] override from ${overridePath}`); - - const override = overridePath - ? (JSON.parse(readFileSync(overridePath, 'utf8')) as Record< - string, - unknown - >) - : {}; - - const config = deepMerge(defaults, override) as IConfig; - - if (!config.version) { - const pkgPath = path.join(PACKAGE_ROOT, 'package.json'); - if (existsSync(pkgPath)) { - try { - const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')) as { - version?: string; - }; - if (pkg.version) config.version = pkg.version; - } catch { - // fall through — /version returns 'unknown' - } - } - } - - // Computed defaults. `origin` and `pub_port` are the externally-visible - // URL+port — what the browser sees. Separate from `port`, which is the - // bind port (can differ when behind a reverse proxy). Code paths that - // build self-referential URLs (GUI bootstrap, email links, OIDC callbacks) - // depend on `origin` having the right port baked in. - if (config.pub_port === undefined) config.pub_port = config.port; - const protocol = config.protocol ?? 'http'; - const domain = config.domain ?? 'localhost'; - const portSuffix = - config.pub_port === 80 || config.pub_port === 443 - ? '' - : `:${config.pub_port}`; - if (config.origin === undefined) { - config.origin = `${protocol}://${domain}${portSuffix}`; - } - // API lives on the `api.` subdomain on the same host+port as the main - // origin (see PuterRouter subdomain handling in server.ts). Compute it - // from pub_port/domain so a single-port override (e.g. port: 5101) flows - // through to the GUI bootstrap without users having to restate the URL. - if (config.api_base_url === undefined) { - config.api_base_url = `${protocol}://api.${domain}${portSuffix}`; - } - - // Resolve path-valued config fields. Two different roots: - // - `extensions` uses RUNTIME_ROOT because extensions ship inside the - // build output (dist/extensions) and the loader's dynamic import() - // resolves relative paths against the *importing* module file. - // - GUI/puter-js/builtin-apps use PACKAGE_ROOT because those assets - // live only in the source tree (not copied into dist/) and are served - // via express.static at runtime. - const resolveRuntime = (p: string): string => - path.isAbsolute(p) ? p : path.resolve(RUNTIME_ROOT, p); - const resolvePackage = (p: string): string => - path.isAbsolute(p) ? p : path.resolve(PACKAGE_ROOT, p); - - if (Array.isArray(config.extensions)) { - config.extensions = config.extensions.map(resolveRuntime); - } - if (typeof config.gui_assets_root === 'string') { - config.gui_assets_root = resolvePackage(config.gui_assets_root); - } - if (typeof config.puterjs_root === 'string') { - config.puterjs_root = resolvePackage(config.puterjs_root); - } - if (isPlainObject(config.builtin_apps)) { - for (const [k, v] of Object.entries(config.builtin_apps)) { - if (typeof v === 'string') { - (config.builtin_apps as Record)[k] = - resolvePackage(v); - } - } - } - return config; -}; - // if called directly, start the server if (require.main === module) { const config = loadConfig(); diff --git a/src/backend/package.json b/src/backend/package.json index bb69cee14..12c898c4f 100644 --- a/src/backend/package.json +++ b/src/backend/package.json @@ -57,8 +57,10 @@ "openai": "^6.34.0", "otpauth": "^9.5.2", "pg": "^8.21.0", + "postal-mime": "^3.0.0", "replicate": "^1.0.0", "sharp": "^0.35.4", + "smtp-server": "^3.19.13", "socket.io": "^4.8.3", "svg-captcha": "^1.4.0", "together-ai": "^0.33.0", @@ -77,6 +79,7 @@ "@types/node": "^24.13.6", "@types/nodemailer": "^8.0.1", "@types/pg": "^8.6.1", + "@types/smtp-server": "^3.5.13", "@types/validator": "^13.15.10", "pgmock": "^1.0.3", "typescript": "^5.9.3", diff --git a/src/backend/services/email/mailbox.test.ts b/src/backend/services/email/mailbox.test.ts new file mode 100644 index 000000000..7c7805064 --- /dev/null +++ b/src/backend/services/email/mailbox.test.ts @@ -0,0 +1,223 @@ +import { Readable } from 'node:stream'; +import { v4 as uuidv4 } from 'uuid'; +import { afterAll, beforeAll, describe, expect, test } from 'vitest'; +import type { PuterServer } from '../../server.js'; +import type { UserRow } from '../../stores/user/UserStore.js'; +import { createTestUser, setupTestServer } from '../../testUtil.js'; +import { + MAIL_CONTENT_TYPE, + copyIntoInbox, + findMailboxOwner, + hasMailbox, + isPuterEmailAddress, + isTempUser, + mailDay, + mailFolderPath, + mailObjectName, + mailboxPath, + puterEmailAddressesOf, + puterEmailUsername, + storeInboxMessage, +} from './mailbox.js'; + +const UUID_V7 = + /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + +// A real in-memory backend rather than stand-in services: these functions +// hand their arguments straight to the filesystem, so a fake would only prove +// which methods were called. +let server: PuterServer; +let dan: UserRow; + +// `frankfurt` is wired to the same bucket the test server already has, so a +// message filed for a Frankfurt account lands somewhere real and the region it +// was placed in can be read back off the entry. +const FRANKFURT_REGION = 'eu-central-1'; + +beforeAll(async () => { + server = await setupTestServer({ + servers: { + frankfurt: { + bucket: 'puter-local', + bucketRegion: FRANKFURT_REGION, + }, + }, + } as never); + await createTestUser(server, { username: 'dan', password: 'secret123!' }); + dan = (await server.stores.user.getByUsername('dan')) as UserRow; +}); + +afterAll(async () => { + await server?.shutdown(); +}); + +describe('addresses', () => { + test('both Puter domains are internal, case-insensitively', () => { + expect(isPuterEmailAddress('dan@puter.email')).toBe(true); + expect(isPuterEmailAddress('Dan@PuterStaging.Email')).toBe(true); + expect(isPuterEmailAddress('dan@apps.puter.email')).toBe(false); + expect(isPuterEmailAddress('dan@example.com')).toBe(false); + expect(isPuterEmailAddress('not-an-address')).toBe(false); + }); + + test('the local part is the username', () => { + expect(puterEmailUsername('dan@puter.email')).toBe('dan'); + }); + + test('an account answers at every Puter domain', () => { + expect(puterEmailAddressesOf('dan')).toEqual([ + 'dan@puter.email', + 'dan@puterstaging.email', + ]); + }); +}); + +describe('account eligibility', () => { + test('temp means no password and no email; either alone is not temp', () => { + expect(isTempUser({ password: null, email: null })).toBe(true); + expect(isTempUser({ password: null, email: 'a@b.c' })).toBe(false); + expect(isTempUser({ password: 'x', email: null })).toBe(false); + }); + + test('the owner of an address is its permanent account, else nothing', async () => { + // A temp account has neither password nor email, which is how one is + // minted before signup completes. + await server.stores.user.create({ + username: 'tmp', + uuid: uuidv4(), + password: null, + email: null, + requires_email_confirmation: false, + }); + + const owner = await findMailboxOwner( + server.stores.user, + 'dan@puter.email', + ); + expect(owner?.username).toBe('dan'); + expect( + await findMailboxOwner(server.stores.user, 'tmp@puter.email'), + ).toBeNull(); + expect( + await findMailboxOwner(server.stores.user, 'nobody@puter.email'), + ).toBeNull(); + }); + + test('a mailbox is set up when ~/.mail is a directory', async () => { + expect(await hasMailbox(server.stores.fsEntry, dan)).toBe(false); + + await server.services.fs.mkdir(dan.id, { + path: mailboxPath(dan.username), + createMissingParents: true, + }); + expect(await hasMailbox(server.stores.fsEntry, dan)).toBe(true); + }); +}); + +describe('layout', () => { + test('folders sit under ~/.mail by UTC day', () => { + expect(mailboxPath('dan')).toBe('/dan/.mail'); + expect(mailFolderPath('dan', 'objects', '2026-09-15')).toBe( + '/dan/.mail/objects/2026-09-15', + ); + expect(mailDay(new Date('2026-09-15T23:59:59.000Z'))).toBe( + '2026-09-15', + ); + }); + + test('an object name is a v7 uuid and the base64url subject', () => { + const [id, encoded] = mailObjectName('Hello, wörld').split('--'); + expect(id).toMatch(UUID_V7); + expect(Buffer.from(encoded, 'base64url').toString('utf8')).toBe( + 'Hello, wörld', + ); + }); + + test('only the first 80 subject characters survive into the name', () => { + const encoded = mailObjectName('x'.repeat(200)).split('--')[1]; + expect(Buffer.from(encoded, 'base64url').toString('utf8')).toBe( + 'x'.repeat(80), + ); + }); + + test('a missing or non-string subject encodes as empty', () => { + expect(mailObjectName(undefined)).toMatch(/--$/); + expect(mailObjectName(['a'])).toMatch(/--$/); + }); +}); + +describe('storeInboxMessage', () => { + test("files under today's inbox folder, in the owner's home region", async () => { + await createTestUser(server, { + username: 'frank', + password: 'secret123!', + }); + const created = (await server.stores.user.getByUsername( + 'frank', + )) as UserRow; + await server.stores.user.update(created.id, { home: 'frankfurt' }); + const owner = (await server.stores.user.getByUsername( + 'frank', + )) as UserRow; + + const entry = await storeInboxMessage(server.services.fs, owner, { + subject: 'hi', + content: Readable.from([Buffer.from('raw')]), + size: 3, + }); + + expect(entry.path).toBe( + `${mailFolderPath('frank', 'objects', mailDay())}/${entry.name}`, + ); + expect(entry.name).toMatch( + new RegExp(`^${UUID_V7.source.slice(1, -1)}--aGk$`), + ); + expect(entry.size).toBe(3); + expect(JSON.parse(entry.metadata as string)).toMatchObject({ + contentType: MAIL_CONTENT_TYPE, + }); + // The message follows the account, rather than landing wherever the + // request happened to arrive. + expect(entry.bucketRegion).toBe(FRANKFURT_REGION); + + const stored = await server.stores.fsEntry.getEntryByPath(entry.path); + expect(stored?.uuid).toBe(entry.uuid); + }); + + test('an explicit day and name are used as given', async () => { + const entry = await storeInboxMessage(server.services.fs, dan, { + content: Buffer.from('raw'), + size: 3, + day: '2020-01-01', + name: 'fixed', + }); + expect(entry.path).toBe('/dan/.mail/objects/2020-01-01/fixed'); + }); +}); + +describe('copyIntoInbox', () => { + test("makes the day folder, then copies under the caller's name", async () => { + const source = await storeInboxMessage(server.services.fs, dan, { + content: Buffer.from('original'), + size: 8, + day: '2020-02-02', + name: 'source', + }); + + const copied = await copyIntoInbox(server.services.fs, dan, { + source, + name: 'copied', + day: '2020-03-03', + }); + + expect(copied.path).toBe('/dan/.mail/objects/2020-03-03/copied'); + // The day folder did not exist before the copy; `copy` cannot create + // it, so the function has to. + expect( + await server.stores.fsEntry.getEntryByPath( + '/dan/.mail/objects/2020-03-03', + ), + ).toBeTruthy(); + expect(copied.size).toBe(source.size); + }); +}); diff --git a/src/backend/services/email/mailbox.ts b/src/backend/services/email/mailbox.ts new file mode 100644 index 000000000..fab655196 --- /dev/null +++ b/src/backend/services/email/mailbox.ts @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import type { Readable } from 'node:stream'; +import type { FSService } from '../fs/FSService.js'; +import { resolveHomeRegion } from '../fs/homeRegion.js'; +import type { FSEntry } from '../../stores/fs/FSEntry.js'; +import type { FSEntryStore } from '../../stores/fs/FSEntryStore.js'; +import type { UserRow, UserStore } from '../../stores/user/UserStore.js'; +import * as uuid from 'uuid'; + +/** + * The user mailbox, shared by everything that fills one: external ingress, + * user-to-user sends and transactional sends. + * + * `@` names an account, and that account's mail lives + * under `//.mail` as `message/rfc822` objects - one folder per UTC + * day, each object named `--` so a listing can sort + * by time and show the subject without opening anything. The SDK's mailbox + * reader is written against this layout, so it is defined once here. + */ + +export const PUTER_EMAIL_DOMAINS = [ + 'puter.email', + 'puterstaging.email', +] as const; + +export const MAIL_CONTENT_TYPE = 'message/rfc822'; + +/** Ceiling on one stored message, inbound or outbound. */ +export const MAX_MESSAGE_BYTES = 25 * 1024 * 1024; + +export type MailFolder = 'objects' | 'sent' | 'outbox'; + +const SUBJECT_NAME_CHARS = 80; + +export const isPuterEmailAddress = (address: string): boolean => + (PUTER_EMAIL_DOMAINS as readonly string[]).includes( + address.split('@')[1]?.toLowerCase() ?? '', + ); + +/** The username a Puter address names. The domain is not checked here. */ +export const puterEmailUsername = (address: string): string => + address.split('@')[0]; + +/** Every address that resolves to `username`. */ +export const puterEmailAddressesOf = (username: string): string[] => + PUTER_EMAIL_DOMAINS.map((domain) => `${username}@${domain}`); + +/** + * A temp account has neither a password nor an email - the same test + * `userProtected` and AuthController use. Read it off a stored user row, never + * off `actor.user`: the actor's copy has `password` stripped, so the check + * there would pass every account. + */ +export const isTempUser = (user: { password?: unknown; email?: unknown }) => + !user.password && !user.email; + +export const mailboxPath = (username: string): string => `/${username}/.mail`; + +export const mailFolderPath = ( + username: string, + folder: MailFolder, + day: string, +): string => `${mailboxPath(username)}/${folder}/${day}`; + +/** `YYYY-MM-DD` in UTC: the day folder a message is filed under. */ +export const mailDay = (date: Date = new Date()): string => + date.toISOString().slice(0, 10); + +/** Object name for one message; only the start of the subject survives. */ +export const mailObjectName = (subject: unknown): string => { + const text = typeof subject === 'string' ? subject : ''; + const encoded = Buffer.from( + text.slice(0, SUBJECT_NAME_CHARS), + 'utf8', + ).toString('base64url'); + return `${uuid.v7()}--${encoded}`; +}; + +/** + * The account behind a Puter address, or null when nothing can receive there: + * no such user, or a temp account (unverified and free to mint, so it has no + * mailbox in either direction). Callers report both as one thing, so mailing an + * address never tells one user which accounts are temp. + */ +export async function findMailboxOwner( + users: UserStore, + address: string, +): Promise { + const user = await users.getByUsername(puterEmailUsername(address)); + return user && !isTempUser(user) ? user : null; +} + +/** Whether the account has set its mailbox up: `~/.mail` exists. */ +export async function hasMailbox( + fsEntries: FSEntryStore, + owner: UserRow, +): Promise { + const entry = await fsEntries.getEntryByPath(mailboxPath(owner.username)); + return entry?.isDir === true; +} + +export interface InboxMessage { + /** Header subject, for the object name. */ + subject?: unknown; + content: Buffer | Readable; + /** Exact byte count; storage needs it before the first byte. */ + size: number; + /** Defaults to today. */ + day?: string; + /** Defaults to a fresh name from `subject`. */ + name?: string; +} + +/** + * File one message in the owner's inbox. Written into the owner's home region + * rather than wherever the request landed, so the mailbox reads locally. + */ +export async function storeInboxMessage( + fs: FSService, + owner: UserRow, + message: InboxMessage, +): Promise { + const day = message.day ?? mailDay(); + const name = message.name ?? mailObjectName(message.subject); + const { fsEntry } = await fs.write( + owner.id, + { + fileMetadata: { + path: `${mailFolderPath(owner.username, 'objects', day)}/${name}`, + size: message.size, + contentType: MAIL_CONTENT_TYPE, + createMissingParents: true, + overwrite: false, + dedupeName: false, + }, + fileContent: message.content, + }, + undefined, + undefined, + resolveHomeRegion(owner), + ); + return fsEntry; +} + +/** + * File an already-stored message in the owner's inbox by copying it, so the + * bytes are never read back. `copy` has no `createMissingParents`, so the day + * folder is made first; `mkdir` is idempotent. + */ +export async function copyIntoInbox( + fs: FSService, + owner: UserRow, + message: { source: FSEntry; name: string; day?: string }, +): Promise { + const parent = await fs.mkdir(owner.id, { + path: mailFolderPath( + owner.username, + 'objects', + message.day ?? mailDay(), + ), + createMissingParents: true, + }); + return await fs.copy(owner.id, { + source: message.source, + destinationParent: parent, + newName: message.name, + }); +} diff --git a/src/backend/smtp/SmtpReceiver.test.ts b/src/backend/smtp/SmtpReceiver.test.ts new file mode 100644 index 000000000..a89f4339e --- /dev/null +++ b/src/backend/smtp/SmtpReceiver.test.ts @@ -0,0 +1,274 @@ +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import SMTPConnection from 'nodemailer/lib/smtp-connection/index.js'; +import { afterAll, afterEach, beforeAll, describe, expect, test } from 'vitest'; +import type { ResolvedSmtpConfig } from './config.js'; +import { SmtpReceiver } from './SmtpReceiver.js'; + +interface IngressCall { + to: string | null; + subject: string | null; + from: string | null; + messageid: string | null; + secret: string | null; + contentLength: string | undefined; + contentType: string | undefined; + body: string; +} + +let ingress: http.Server; +let ingressUrl: string; +let calls: IngressCall[] = []; +let statusFor: (to: string) => number = () => 200; + +let receiver: SmtpReceiver; +let smtpPort: number; + +const baseConfig = ( + over: Partial = {}, +): ResolvedSmtpConfig => ({ + host: '127.0.0.1', + port: 0, + hostname: 'mx.example.com', + domains: ['example.com'], + ingressUrl, + ingressHost: null, + secret: 'ingress-secret', + maxMessageBytes: 2048, + maxRecipients: 3, + maxClients: 20, + ...over, +}); + +const startReceiver = async (over: Partial = {}) => { + receiver = new SmtpReceiver(baseConfig(over)); + smtpPort = (await receiver.listen()).port; +}; + +/** Deliver one message and resolve with the SMTP reply code. */ +const deliver = async (opts: { + to: string[]; + message?: string; +}): Promise<{ code: number; message: string }> => { + const conn = new SMTPConnection({ + host: '127.0.0.1', + port: smtpPort, + secure: false, + ignoreTLS: true, + tls: { rejectUnauthorized: false }, + }); + return await new Promise((resolve, reject) => { + conn.on('error', reject); + conn.connect(() => { + conn.send( + { + from: 'sender@elsewhere.test', + to: opts.to, + }, + opts.message ?? + 'Subject: Hello\r\nFrom: sender@elsewhere.test\r\n\r\nbody\r\n', + (err: (Error & { responseCode?: number }) | null) => { + conn.close(); + if (err) { + resolve({ + code: err.responseCode ?? 0, + message: err.message, + }); + return; + } + resolve({ code: 250, message: 'ok' }); + }, + ); + }); + }); +}; + +/** + * Drive the server with raw SMTP. nodemailer refuses an oversize declaration + * client-side, which would never reach the server being tested. + */ +const rawExchange = async (commands: string[]): Promise => { + const { createConnection } = await import('node:net'); + return await new Promise((resolve, reject) => { + const replies: string[] = []; + const socket = createConnection(smtpPort, '127.0.0.1'); + let next = 0; + socket.setEncoding('utf8'); + socket.on('data', (chunk: string) => { + replies.push(chunk.trim()); + if (next < commands.length) { + socket.write(`${commands[next++]}\r\n`); + } else { + socket.end(); + } + }); + socket.on('error', reject); + socket.on('close', () => resolve(replies)); + }); +}; + +beforeAll(async () => { + ingress = http.createServer((req, res) => { + const url = new URL(req.url ?? '', 'http://ingress.test'); + const chunks: Buffer[] = []; + req.on('data', (c) => chunks.push(c as Buffer)); + req.on('end', () => { + const to = url.searchParams.get('to') ?? ''; + calls.push({ + to, + subject: url.searchParams.get('subject'), + from: url.searchParams.get('from'), + messageid: url.searchParams.get('messageid'), + secret: url.searchParams.get('SECRET'), + contentLength: req.headers['content-length'], + contentType: req.headers['content-type'], + body: Buffer.concat(chunks).toString(), + }); + res.statusCode = statusFor(to); + res.end(); + }); + }); + await new Promise((r) => ingress.listen(0, '127.0.0.1', r)); + ingressUrl = `http://127.0.0.1:${(ingress.address() as AddressInfo).port}/email/ingress`; + await startReceiver(); +}); + +afterAll(async () => { + await receiver.close(); + await new Promise((r) => ingress.close(() => r())); +}); + +afterEach(() => { + calls = []; + statusFor = () => 200; +}); + +describe('accepting mail', () => { + test('delivers a message and reports 250', async () => { + const reply = await deliver({ to: ['dan@example.com'] }); + expect(reply.code).toBe(250); + expect(calls).toHaveLength(1); + expect(calls[0].to).toBe('dan@example.com'); + expect(calls[0].subject).toBe('Hello'); + expect(calls[0].from).toBe('sender@elsewhere.test'); + expect(calls[0].secret).toBe('ingress-secret'); + expect(calls[0].contentType).toBe('message/rfc822'); + expect(calls[0].contentLength).toBe( + String(Buffer.byteLength(calls[0].body)), + ); + }); + + test('sends NO-ID when the message has no Message-ID', async () => { + await deliver({ to: ['dan@example.com'] }); + expect(calls[0].messageid).toBe('NO-ID'); + }); + + test('posts once per envelope recipient with identical bytes', async () => { + const reply = await deliver({ + to: ['dan@example.com', 'sam@example.com'], + }); + expect(reply.code).toBe(250); + expect(calls.map((c) => c.to).sort()).toEqual([ + 'dan@example.com', + 'sam@example.com', + ]); + expect(calls[0].body).toBe(calls[1].body); + }); + + test('delivers to the envelope recipient, not the To: header', async () => { + // A blind-copied recipient never appears in the To: header, so reading + // the header instead of the envelope misdelivers the message. + await deliver({ + to: ['hidden@example.com'], + message: + 'Subject: Bcc test\r\nTo: someone-else@example.com\r\n\r\nbody\r\n', + }); + expect(calls).toHaveLength(1); + expect(calls[0].to).toBe('hidden@example.com'); + }); + + test('delivers one copy when an address is repeated', async () => { + await deliver({ to: ['dan@example.com', 'dan@example.com'] }); + expect(calls).toHaveLength(1); + }); +}); + +describe('refusing recipients', () => { + test('refuses a domain it does not accept, and posts nothing', async () => { + const reply = await deliver({ to: ['someone@elsewhere.test'] }); + expect(reply.code).toBe(550); + expect(reply.message).toContain('Relay access denied'); + expect(calls).toHaveLength(0); + }); + + test('refuses more recipients than the cap allows', async () => { + const reply = await deliver({ + to: [ + 'a@example.com', + 'b@example.com', + 'c@example.com', + 'd@example.com', + ], + }); + // The first three are accepted, so the message still goes through. + expect(reply.code).toBe(250); + expect(calls).toHaveLength(3); + }); +}); + +describe('message size', () => { + test('refuses before the body when the sender declares an oversize', async () => { + const replies = await rawExchange([ + 'EHLO test.local', + 'MAIL FROM: SIZE=99999999', + 'QUIT', + ]); + expect(replies.join('\n')).toMatch(/^552/m); + expect(calls).toHaveLength(0); + }); + + test('advertises the SIZE limit so senders can check before sending', async () => { + const replies = await rawExchange(['EHLO test.local', 'QUIT']); + expect(replies.join('\n')).toContain('SIZE 2048'); + }); + + test('refuses an oversize body that was never declared', async () => { + const big = `Subject: big\r\n\r\n${'x'.repeat(4096)}\r\n`; + const reply = await deliver({ to: ['dan@example.com'], message: big }); + expect(reply.code).toBe(552); + expect(calls).toHaveLength(0); + }); +}); + +describe('reporting what the ingress endpoint said', () => { + test('an unknown user becomes a permanent 550', async () => { + statusFor = () => 404; + const reply = await deliver({ to: ['nobody@example.com'] }); + expect(reply.code).toBe(550); + }); + + test('a rejected secret becomes a temporary 451', async () => { + statusFor = () => 403; + const reply = await deliver({ to: ['dan@example.com'] }); + expect(reply.code).toBe(451); + }); + + test('an unreachable endpoint becomes a temporary 451', async () => { + await receiver.close(); + await startReceiver({ ingressUrl: 'http://127.0.0.1:1/email/ingress' }); + const reply = await deliver({ to: ['dan@example.com'] }); + expect(reply.code).toBe(451); + await receiver.close(); + await startReceiver(); + }); + + test('reports the first failure when recipients differ', async () => { + // One reply covers the whole message; the sender is told about the + // first address that could not be delivered to. + statusFor = (to) => (to.startsWith('nobody') ? 404 : 200); + const reply = await deliver({ + to: ['dan@example.com', 'nobody@example.com'], + }); + expect(reply.code).toBe(550); + }); +}); diff --git a/src/backend/smtp/SmtpReceiver.ts b/src/backend/smtp/SmtpReceiver.ts new file mode 100644 index 000000000..50964116c --- /dev/null +++ b/src/backend/smtp/SmtpReceiver.ts @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { SMTPServer } from 'smtp-server'; +import type { SMTPServerSession } from 'smtp-server'; +import type { ResolvedSmtpConfig } from './config.js'; +import { + ACCEPTED, + type SmtpReply, + deliver, + describeTarget, + readHeaderFields, +} from './ingressClient.js'; +import { acceptedRecipients } from './recipients.js'; + +const SOCKET_TIMEOUT_MS = 60_000; +const INGRESS_TIMEOUT_MS = 120_000; + +const replyError = (reply: SmtpReply): Error => + Object.assign(new Error(reply.text), { responseCode: reply.code }); + +/** + * An SMTP listener that hands each message to the mail ingress endpoint over + * HTTP. It stores nothing and never relays: the only addresses it accepts are + * in a configured domain, and the only place a message goes is that endpoint. + * + * Messages are held in memory while they are delivered, which is why + * `maxClients` matters - it bounds concurrent messages, and so memory, at + * roughly `maxClients` x the 25MiB message ceiling. + */ +export class SmtpReceiver { + readonly #server: SMTPServer; + + constructor(private readonly cfg: ResolvedSmtpConfig) { + this.#server = new SMTPServer({ + name: cfg.hostname, + // Advertises SIZE, so an oversized message is refused at MAIL FROM + // rather than after it has crossed the wire. + size: cfg.maxMessageBytes, + // Mail arriving from the internet authenticates nobody; the gate is + // the accepted-domain check below. + disabledCommands: ['AUTH'], + authOptional: true, + hideSTARTTLS: true, + maxClients: cfg.maxClients, + socketTimeout: SOCKET_TIMEOUT_MS, + disableReverseLookup: true, + logger: false, + onRcptTo: (address, session, callback) => { + const [recipient] = acceptedRecipients( + [address.address], + cfg.domains, + ); + if (!recipient) { + // Refusing every other domain is what keeps this from being + // an open relay. + callback( + replyError({ + code: 550, + text: '5.7.1 Relay access denied', + }), + ); + return; + } + if (session.envelope.rcptTo.length >= cfg.maxRecipients) { + callback( + replyError({ + code: 452, + text: '4.5.3 Too many recipients', + }), + ); + return; + } + callback(); + }, + onData: (stream, session, callback) => { + this.#handle(stream, session) + .then((reply) => + callback(reply.code === 250 ? null : replyError(reply)), + ) + .catch(() => + callback( + replyError({ + code: 451, + text: '4.3.0 Temporary server error', + }), + ), + ); + }, + }); + } + + async #handle( + stream: NodeJS.ReadableStream & { sizeExceeded?: boolean }, + session: SMTPServerSession, + ): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) chunks.push(chunk as Buffer); + if (stream.sizeExceeded) { + return { code: 552, text: '5.3.4 Message too big for system' }; + } + const message = Buffer.concat(chunks); + + const fields = await readHeaderFields(message); + // The envelope recipient, not the To: header - a blind-copied address + // never appears in the header at all. + const recipients = acceptedRecipients( + session.envelope.rcptTo.map((r) => r.address), + this.cfg.domains, + ); + + for (const recipient of recipients) { + const reply = await deliver(message, recipient, fields, { + url: this.cfg.ingressUrl, + host: this.cfg.ingressHost, + secret: this.cfg.secret, + timeoutMs: INGRESS_TIMEOUT_MS, + }); + if (reply.code !== 250) { + // One reply covers the whole message, so the first failure is + // what the sender is told. + console.warn('[smtp] delivery refused', { + session: session.id, + recipient, + code: reply.code, + // Never the full URL: the secret is in its query string. + ingress: describeTarget(this.cfg.ingressUrl), + }); + return reply; + } + } + return ACCEPTED; + } + + async listen(): Promise<{ port: number }> { + await new Promise((resolve, reject) => { + this.#server.once('error', reject); + this.#server.listen(this.cfg.port, this.cfg.host, () => { + this.#server.off('error', reject); + resolve(); + }); + }); + const address = this.#server.server.address(); + return { + port: typeof address === 'object' && address ? address.port : 0, + }; + } + + async close(): Promise { + await new Promise((resolve) => this.#server.close(resolve)); + } +} diff --git a/src/backend/smtp/config.test.ts b/src/backend/smtp/config.test.ts new file mode 100644 index 000000000..3f30c936b --- /dev/null +++ b/src/backend/smtp/config.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, test } from 'vitest'; +import type { IConfig } from '../types'; +import { + SmtpConfigError, + isLocalServerEnabled, + resolveSmtpConfig, +} from './config.js'; + +const config = (userEmail: Record | undefined): IConfig => + ({ + api_base_url: 'https://api.example.com', + userEmail, + }) as unknown as IConfig; + +const valid = { + secret: 'ingress-secret', + localServer: true, + localDomains: ['example.com'], +}; + +describe('deciding whether the receiver runs', () => { + test('runs only when localServer is exactly true', () => { + expect(isLocalServerEnabled(config({ ...valid }))).toBe(true); + }); + + test.each([ + ['absent userEmail', undefined], + ['no localServer', { secret: 's' }], + ['localServer false', { secret: 's', localServer: false }], + ['a truthy non-boolean', { secret: 's', localServer: 'yes' }], + ])('does not run with %s', (_label, userEmail) => { + expect( + isLocalServerEnabled(config(userEmail as Record)), + ).toBe(false); + }); +}); + +describe('resolving receiver config', () => { + test('derives the ingress url from api_base_url', () => { + expect(resolveSmtpConfig(config(valid)).ingressUrl).toBe( + 'https://api.example.com/email/ingress', + ); + }); + + test('an explicit ingress url wins', () => { + const cfg = resolveSmtpConfig( + config({ ...valid, localIngressUrl: 'http://ingress.test/x' }), + ); + expect(cfg.ingressUrl).toBe('http://ingress.test/x'); + }); + + test('addresses the api virtual host by default', () => { + // The ingress route is served on the api subdomain, so a request has + // to name that host even when it connects somewhere else. + expect(resolveSmtpConfig(config(valid)).ingressHost).toBe( + 'api.example.com', + ); + }); + + test('keeps the api host when the url points at an internal address', () => { + const cfg = resolveSmtpConfig( + config({ + ...valid, + localIngressUrl: 'http://puter:4100/email/ingress', + }), + ); + expect(cfg.ingressUrl).toBe('http://puter:4100/email/ingress'); + expect(cfg.ingressHost).toBe('api.example.com'); + }); + + test('an explicit ingress host wins', () => { + const cfg = resolveSmtpConfig( + config({ ...valid, localIngressHost: 'api.internal' }), + ); + expect(cfg.ingressHost).toBe('api.internal'); + }); + + test('defaults the port to 2525 so no privileged bind is needed', () => { + expect(resolveSmtpConfig(config(valid)).port).toBe(2525); + }); + + test('port 25 can be opted into', () => { + expect( + resolveSmtpConfig(config({ ...valid, localPort: 25 })).port, + ).toBe(25); + }); + + test('lowercases accepted domains', () => { + const cfg = resolveSmtpConfig( + config({ ...valid, localDomains: ['Example.COM'] }), + ); + expect(cfg.domains).toEqual(['example.com']); + }); + + test('names the first domain as the greeting hostname', () => { + expect(resolveSmtpConfig(config(valid)).hostname).toBe('example.com'); + }); + + test('applies the documented defaults', () => { + expect(resolveSmtpConfig(config(valid))).toMatchObject({ + host: '0.0.0.0', + maxRecipients: 50, + maxClients: 20, + }); + }); + + test('takes the message cap from the mailbox module, not config', () => { + // Sharing the constant is what stops the receiver accepting a message + // the ingress endpoint would then refuse. + const cfg = resolveSmtpConfig( + config({ ...valid, maxMessageBytes: 999 }), + ); + expect(cfg.maxMessageBytes).toBe(25 * 1024 * 1024); + }); +}); + +describe('refusing an unusable config', () => { + test('requires a secret', () => { + expect(() => + resolveSmtpConfig( + config({ localServer: true, localDomains: ['e.test'] }), + ), + ).toThrow(SmtpConfigError); + }); + + test('requires at least one domain, so it cannot become an open relay', () => { + expect(() => + resolveSmtpConfig(config({ secret: 's', localServer: true })), + ).toThrow(/localDomains/); + }); + + test('rejects a non-absolute ingress url', () => { + expect(() => + resolveSmtpConfig( + config({ ...valid, localIngressUrl: '/email/ingress' }), + ), + ).toThrow(/absolute URL/); + }); + + test('rejects an impossible port', () => { + expect(() => + resolveSmtpConfig(config({ ...valid, localPort: 70000 })), + ).toThrow(/localPort/); + }); + + test('names every problem at once', () => { + try { + resolveSmtpConfig(config({ localServer: true })); + expect.unreachable('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(SmtpConfigError); + expect((err as SmtpConfigError).problems.length).toBeGreaterThan(1); + } + }); +}); diff --git a/src/backend/smtp/config.ts b/src/backend/smtp/config.ts new file mode 100644 index 000000000..8284c496e --- /dev/null +++ b/src/backend/smtp/config.ts @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { MAX_MESSAGE_BYTES } from '../services/email/mailbox.js'; +import type { IConfig } from '../types'; + +export interface ResolvedSmtpConfig { + host: string; + port: number; + hostname: string; + domains: string[]; + ingressUrl: string; + /** Host header override; the ingress route is gated on the api subdomain. */ + ingressHost: string | null; + secret: string; + maxMessageBytes: number; + maxRecipients: number; + maxClients: number; +} + +export class SmtpConfigError extends Error { + constructor(public readonly problems: string[]) { + super(`invalid userEmail receiver config:\n- ${problems.join('\n- ')}`); + this.name = 'SmtpConfigError'; + } +} + +/** Whether the local receiver is switched on at all. */ +export const isLocalServerEnabled = (config: IConfig): boolean => + config.userEmail?.localServer === true; + +/** + * Apply defaults and refuse an unusable configuration, naming every problem at + * once. A listener that starts and then rejects every message is harder to + * diagnose than one that will not start. + */ +export const resolveSmtpConfig = (config: IConfig): ResolvedSmtpConfig => { + const cfg = config.userEmail ?? {}; + const problems: string[] = []; + + const secret = typeof cfg.secret === 'string' ? cfg.secret : ''; + if (!secret) problems.push('userEmail.secret must be set'); + + const domains = Array.isArray(cfg.localDomains) + ? cfg.localDomains + .filter((d): d is string => typeof d === 'string' && d.length > 0) + .map((d) => d.toLowerCase()) + : []; + if (domains.length === 0) { + problems.push( + 'userEmail.localDomains must list at least one domain to accept mail for', + ); + } + + const ingressUrl = + cfg.localIngressUrl || + (config.api_base_url + ? `${config.api_base_url.replace(/\/$/, '')}/email/ingress` + : ''); + if (!ingressUrl) { + problems.push( + 'userEmail.localIngressUrl must be set when api_base_url is not configured', + ); + } else { + try { + new URL(ingressUrl); + } catch { + problems.push('userEmail.localIngressUrl must be an absolute URL'); + } + } + + const port = cfg.localPort ?? 2525; + if (!Number.isInteger(port) || port < 1 || port > 65535) { + problems.push('userEmail.localPort must be a port number'); + } + + // The ingress route is served on the api subdomain, so a URL pointing at an + // internal address still has to address that virtual host by name. + let ingressHost = cfg.localIngressHost ?? null; + if (!ingressHost && config.api_base_url) { + try { + ingressHost = new URL(config.api_base_url).host; + } catch { + ingressHost = null; + } + } + + if (problems.length > 0) throw new SmtpConfigError(problems); + + return { + host: cfg.localHost ?? '0.0.0.0', + port, + hostname: cfg.localHostname ?? domains[0], + domains, + ingressUrl, + ingressHost, + secret, + maxMessageBytes: MAX_MESSAGE_BYTES, + maxRecipients: cfg.localMaxRecipients ?? 50, + // Bounds concurrent messages, and so memory, since each is held whole. + maxClients: cfg.localMaxClients ?? 20, + }; +}; diff --git a/src/backend/smtp/index.ts b/src/backend/smtp/index.ts new file mode 100644 index 000000000..a65072323 --- /dev/null +++ b/src/backend/smtp/index.ts @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { loadConfig } from '../config'; +import { SmtpReceiver } from './SmtpReceiver.js'; +import { + SmtpConfigError, + isLocalServerEnabled, + resolveSmtpConfig, +} from './config.js'; +import { describeTarget } from './ingressClient.js'; + +// Its own process rather than part of the server: it binds its own port, and a +// fault here should not take the API down with it. +if (require.main === module) { + const config = loadConfig(); + + if (!isLocalServerEnabled(config)) { + console.log( + '[smtp] userEmail.localServer is not enabled - nothing to run', + ); + process.exit(0); + } + + let cfg; + try { + cfg = resolveSmtpConfig(config); + } catch (err) { + if (!(err instanceof SmtpConfigError)) throw err; + console.error(`[smtp] ${err.message}`); + process.exit(1); + } + + const receiver = new SmtpReceiver(cfg); + receiver + .listen() + .then(({ port }) => { + console.log( + `[smtp] accepting mail for ${cfg.domains.join(', ')} on ${cfg.host}:${port}`, + ); + console.log( + `[smtp] forwarding to ${describeTarget(cfg.ingressUrl)}`, + ); + }) + .catch((err: Error) => { + console.error(`[smtp] failed to listen: ${err.message}`); + process.exit(1); + }); + + const shutDown = async () => { + await receiver.close(); + process.exit(0); + }; + process.on('SIGINT', shutDown); + process.on('SIGTERM', shutDown); +} diff --git a/src/backend/smtp/ingressClient.test.ts b/src/backend/smtp/ingressClient.test.ts new file mode 100644 index 000000000..ec8f9d00c --- /dev/null +++ b/src/backend/smtp/ingressClient.test.ts @@ -0,0 +1,205 @@ +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { afterAll, afterEach, beforeAll, describe, expect, test } from 'vitest'; +import { + buildIngressUrl, + deliver, + describeTarget, + readHeaderFields, + replyForStatus, +} from './ingressClient.js'; + +const SECRET = 'ingress-secret'; + +describe('building the request url', () => { + const url = (over: Partial[1]> = {}) => + buildIngressUrl('https://api.example.com/email/ingress', { + to: 'dan@example.com', + subject: 'Hi & bye', + from: 'sender@elsewhere.test', + messageId: '', + secret: SECRET, + ...over, + }); + + test('carries the metadata the endpoint reads', () => { + const u = url(); + expect(u.searchParams.get('to')).toBe('dan@example.com'); + expect(u.searchParams.get('subject')).toBe('Hi & bye'); + expect(u.searchParams.get('from')).toBe('sender@elsewhere.test'); + expect(u.searchParams.get('messageid')).toBe(''); + expect(u.searchParams.get('SECRET')).toBe(SECRET); + }); + + test('encodes a unicode subject', () => { + const u = url({ subject: 'héllo → wörld' }); + expect(u.searchParams.get('subject')).toBe('héllo → wörld'); + expect(u.toString()).not.toContain('→'); + }); +}); + +describe('describing the target for a log', () => { + test('never includes the secret', () => { + const described = describeTarget( + `https://api.example.com/email/ingress?SECRET=${SECRET}`, + ); + expect(described).toBe('https://api.example.com/email/ingress'); + expect(described).not.toContain(SECRET); + }); + + test('does not throw on an unusable url', () => { + expect(describeTarget('not a url')).not.toContain('not a url'); + }); +}); + +describe('reading header fields', () => { + test('reads subject, from and message id', async () => { + const fields = await readHeaderFields( + Buffer.from( + 'Subject: Hello there\r\nFrom: Dan \r\nMessage-ID: \r\n\r\nbody', + ), + ); + expect(fields).toEqual({ + subject: 'Hello there', + from: 'dan@example.com', + messageId: '', + }); + }); + + test('decodes an encoded-word subject', () => { + // The subject becomes part of the stored object name, so it has to be + // decoded the same way the hosted receiver's pre-parsed header was. + return expect( + readHeaderFields( + Buffer.from('Subject: =?UTF-8?B?SGVsbG8gd29ybGQ=?=\r\n\r\nx'), + ), + ).resolves.toMatchObject({ subject: 'Hello world' }); + }); + + test('falls back the way the hosted receiver does', async () => { + expect(await readHeaderFields(Buffer.from('X-Other: v\r\n\r\nx'))).toEqual( + { + subject: '', + from: 'anonymous@example.com', + messageId: 'NO-ID', + }, + ); + }); +}); + +describe('turning a status into an SMTP reply', () => { + test.each([ + [200, 250], + [204, 250], + [404, 550], + [413, 552], + [403, 451], + [411, 451], + [429, 451], + [500, 451], + ])('%i becomes %i', (status, code) => { + expect(replyForStatus(status).code).toBe(code); + }); + + test('a rejected secret is temporary, so mail is held not bounced', () => { + // 403 means this end is misconfigured; bouncing would discard the mail. + expect(replyForStatus(403).code).toBe(451); + }); +}); + +describe('delivering to the endpoint', () => { + let server: http.Server; + let base: string; + let received: Array<{ + url: string; + headers: http.IncomingHttpHeaders; + body: Buffer; + }> = []; + let status = 200; + let hang = false; + + beforeAll(async () => { + server = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on('data', (c) => chunks.push(c as Buffer)); + req.on('end', () => { + received.push({ + url: req.url ?? '', + headers: req.headers, + body: Buffer.concat(chunks), + }); + if (hang) return; + res.statusCode = status; + res.end(); + }); + }); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + base = `http://127.0.0.1:${(server.address() as AddressInfo).port}/email/ingress`; + }); + + afterAll(async () => { + await new Promise((r) => server.close(() => r())); + }); + + afterEach(() => { + received = []; + status = 200; + hang = false; + }); + + const fields = { + subject: 'Hi', + from: 'sender@elsewhere.test', + messageId: '', + }; + const send = (target: Partial<{ url: string; host: string | null }> = {}) => + deliver(Buffer.from('Subject: Hi\r\n\r\nthe body'), 'dan@example.com', fields, { + url: base, + secret: SECRET, + timeoutMs: 2000, + ...target, + }); + + test('posts the message with an exact content length', async () => { + const reply = await send(); + expect(reply.code).toBe(250); + expect(received).toHaveLength(1); + expect(received[0].headers['content-type']).toBe('message/rfc822'); + expect(received[0].headers['content-length']).toBe( + String(received[0].body.byteLength), + ); + expect(received[0].body.toString()).toBe('Subject: Hi\r\n\r\nthe body'); + expect(received[0].url).toContain('to=dan%40example.com'); + }); + + test('sends the configured host so the api vhost is addressed', async () => { + await send({ host: 'api.example.com' }); + expect(received[0].headers.host).toBe('api.example.com'); + }); + + test('falls back to the url host when none is configured', async () => { + await send(); + expect(received[0].headers.host).toContain('127.0.0.1'); + }); + + test('maps a refusal from the endpoint', async () => { + status = 404; + expect((await send()).code).toBe(550); + }); + + test('treats an unreachable endpoint as temporary', async () => { + const reply = await send({ url: 'http://127.0.0.1:1/email/ingress' }); + expect(reply.code).toBe(451); + }); + + test('treats a hung endpoint as temporary', async () => { + hang = true; + const reply = await deliver( + Buffer.from('x'), + 'dan@example.com', + fields, + { url: base, secret: SECRET, timeoutMs: 150 }, + ); + expect(reply.code).toBe(451); + }); +}); diff --git a/src/backend/smtp/ingressClient.ts b/src/backend/smtp/ingressClient.ts new file mode 100644 index 000000000..d5e2521fe --- /dev/null +++ b/src/backend/smtp/ingressClient.ts @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import http from 'node:http'; +import https from 'node:https'; +import PostalMime from 'postal-mime'; +import { MAIL_CONTENT_TYPE } from '../services/email/mailbox.js'; + +/** + * An SMTP reply: 250 when the endpoint took the message, an error code + * otherwise. + */ +export interface SmtpReply { + code: number; + text: string; +} + +export const ACCEPTED: SmtpReply = { + code: 250, + text: '2.0.0 Message accepted', +}; + +export interface IngressTarget { + url: string; + /** Sent as the Host header: the endpoint is served on the api subdomain. */ + host?: string | null; + secret: string; + timeoutMs: number; +} + +/** + * Header fields the endpoint takes in its query string. Fallbacks match what + * the hosted receiver sends, so a message looks the same whichever one + * delivered it. + */ +export const readHeaderFields = async (message: Buffer) => { + try { + const parsed = await PostalMime.parse(message); + return { + subject: parsed.subject ?? '', + from: parsed.from?.address ?? 'anonymous@example.com', + messageId: parsed.messageId ?? 'NO-ID', + }; + } catch { + // A message we cannot parse is still a message we can deliver. + return { + subject: '', + from: 'anonymous@example.com', + messageId: 'NO-ID', + }; + } +}; + +export const buildIngressUrl = ( + base: string, + params: { + to: string; + subject: string; + from: string; + messageId: string; + secret: string; + }, +): URL => { + const url = new URL(base); + url.searchParams.set('subject', params.subject); + url.searchParams.set('messageid', params.messageId); + url.searchParams.set('from', params.from); + url.searchParams.set('to', params.to); + url.searchParams.set('SECRET', params.secret); + return url; +}; + +/** + * The endpoint without its query string. The secret travels as a query + * parameter, so the full URL is a credential and only this form may be logged. + */ +export const describeTarget = (base: string): string => { + try { + const url = new URL(base); + return `${url.origin}${url.pathname}`; + } catch { + return ''; + } +}; + +/** + * Anything other than "stored" and "no such user" is reported as temporary, so + * the sending server retries rather than bouncing mail over a problem at this + * end - a wrong secret included. + */ +export const replyForStatus = (status: number): SmtpReply => { + if (status >= 200 && status < 300) return ACCEPTED; + if (status === 404) return { code: 550, text: '5.1.1 No such user here' }; + if (status === 413) + return { code: 552, text: '5.3.4 Message too big for system' }; + return { code: 451, text: '4.3.0 Temporary server error' }; +}; + +/** Hand one message to the ingress endpoint for one recipient. */ +export const deliver = async ( + message: Buffer, + recipient: string, + fields: { subject: string; from: string; messageId: string }, + target: IngressTarget, +): Promise => { + const url = buildIngressUrl(target.url, { + to: recipient, + ...fields, + secret: target.secret, + }); + const transport = url.protocol === 'https:' ? https : http; + + return await new Promise((resolve) => { + const req = transport.request( + url, + { + method: 'POST', + headers: { + 'content-type': MAIL_CONTENT_TYPE, + // Required: the endpoint refuses a body of unknown length. + 'content-length': String(message.byteLength), + ...(target.host ? { host: target.host } : {}), + }, + timeout: target.timeoutMs, + }, + (res) => { + res.resume(); + res.once('end', () => + resolve(replyForStatus(res.statusCode ?? 0)), + ); + }, + ); + // Errors from the HTTP client can carry the request URL, and that URL + // holds the secret, so none of it is surfaced. + req.once('error', () => + resolve({ code: 451, text: '4.4.1 Ingress unavailable' }), + ); + req.once('timeout', () => { + req.destroy(); + resolve({ code: 451, text: '4.4.2 Ingress timed out' }); + }); + req.end(message); + }); +}; diff --git a/src/backend/smtp/recipients.test.ts b/src/backend/smtp/recipients.test.ts new file mode 100644 index 000000000..732942c56 --- /dev/null +++ b/src/backend/smtp/recipients.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, test } from 'vitest'; +import { acceptedRecipients } from './recipients.js'; + +const DOMAINS = ['example.com', 'Mail.Example.Net']; +const accept = (...addresses: string[]) => + acceptedRecipients(addresses, DOMAINS); + +describe('choosing which recipients to accept', () => { + test('accepts a configured domain regardless of case', () => { + expect(accept('dan@example.com', 'sam@MAIL.EXAMPLE.NET')).toEqual([ + 'dan@example.com', + 'sam@MAIL.EXAMPLE.NET', + ]); + }); + + test('refuses every other domain, which is what stops open relaying', () => { + expect( + accept( + 'someone@elsewhere.test', + 'sub@sub.example.com', + 'spoof@example.com.evil.test', + ), + ).toEqual([]); + }); + + test('accepts nothing when no domains are configured', () => { + expect(acceptedRecipients(['dan@example.com'], [])).toEqual([]); + }); + + test('keeps the local part verbatim', () => { + // What a local part names is the ingress endpoint's decision, so it is + // handed over exactly as the sender wrote it. + expect(accept('Dan.Smith+tag@Example.COM')).toEqual([ + 'Dan.Smith+tag@Example.COM', + ]); + }); + + test('splits on the last @, which a quoted local part may contain', () => { + expect(accept('"odd@name"@example.com')).toEqual([ + '"odd@name"@example.com', + ]); + }); + + test.each([ + ['no at sign', 'not-an-address'], + ['empty local part', '@example.com'], + ['empty domain', 'dan@'], + ['whitespace', 'dan smith@example.com'], + ['empty string', ''], + ])('skips a malformed address (%s)', (_label, address) => { + expect(accept(address)).toEqual([]); + }); + + test('preserves order and drops repeats', () => { + expect( + accept('a@example.com', 'b@example.com', 'a@example.com'), + ).toEqual(['a@example.com', 'b@example.com']); + }); + + test('treats a domain-case variant as the same recipient', () => { + expect(accept('a@example.com', 'a@EXAMPLE.COM')).toEqual([ + 'a@example.com', + ]); + }); +}); diff --git a/src/backend/smtp/recipients.ts b/src/backend/smtp/recipients.ts new file mode 100644 index 000000000..2f9ef33a3 --- /dev/null +++ b/src/backend/smtp/recipients.ts @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * The addresses this server will take mail for, in the order offered and + * without repeats. + * + * An address is accepted only when its domain is one of `domains` - that check + * is the whole reason this server is not an open relay. The local part is + * passed through exactly as the sender wrote it, because what it names is the + * ingress endpoint's decision, not this one's. + */ +export const acceptedRecipients = ( + addresses: readonly string[], + domains: readonly string[], +): string[] => { + const accepted = domains.map((d) => d.toLowerCase()); + const seen = new Set(); + const out: string[] = []; + + for (const address of addresses) { + const at = address.lastIndexOf('@'); + if (at <= 0 || at === address.length - 1) continue; + if (/\s/.test(address)) continue; + + const domain = address.slice(at + 1).toLowerCase(); + if (!accepted.includes(domain)) continue; + + const key = `${address.slice(0, at)}@${domain}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(address); + } + return out; +}; diff --git a/src/backend/types.ts b/src/backend/types.ts index e1fcd97d9..c50bd905f 100644 --- a/src/backend/types.ts +++ b/src/backend/types.ts @@ -181,6 +181,46 @@ export interface IEmailConfig { [key: string]: unknown; } +/** + * Puter-addressed user mailboxes: the shared secret the ingress endpoint + * checks, and the optional local SMTP receiver that feeds it. + * + * The receiver is the self-hosted stand-in for an externally-hosted inbound + * mail path — `localServer` switches it on the same way `workers.localServer` + * switches workers to an in-process implementation. It runs as its own process, + * so nothing here affects the main server when it is off. + */ +export interface IUserEmailConfig { + /** Shared secret the ingress endpoint requires. Unset disables ingress. */ + secret?: string; + /** Run the local SMTP receiver. Absent or falsy and nothing binds. */ + localServer?: boolean; + /** Bind port. Default 2525 — set 25 only where the process may bind it. */ + localPort?: number; + /** Bind address. Default '0.0.0.0'. */ + localHost?: string; + /** Domains to accept recipients for. Required with `localServer`. */ + localDomains?: string[]; + /** Greeting and EHLO name. Defaults to the first accepted domain. */ + localHostname?: string; + /** Ingress URL. Defaults to `/email/ingress`. */ + localIngressUrl?: string; + /** + * Host header to send. The ingress route is served on the `api` subdomain, + * so this must name it when `localIngressUrl` points at an internal + * address. Defaults to the host of `api_base_url`. + */ + localIngressHost?: string; + /** Envelope recipients accepted per message. Default 50. */ + localMaxRecipients?: number; + /** + * Concurrent client connections. Default 20. Each message is held in memory + * while it is delivered, so this also bounds memory use. + */ + localMaxClients?: number; + [key: string]: unknown; +} + /** Prelude (https://prelude.so) Verify v2 — SMS phone verification provider. */ export interface IPreludeConfig { /** Prelude v2 API key (sent as `Authorization: Bearer `). */ @@ -941,6 +981,8 @@ interface IConfigOptional { kvCache: IKvCacheConfig; pager: IPagerConfig; email: IEmailConfig; + /** Puter-addressed user mailboxes and the optional local SMTP receiver. */ + userEmail: IUserEmailConfig; /** Optional — only set when SMS phone verification (Prelude) is wired in. */ prelude: IPreludeConfig; /** Optional — only set when a ClickHouse analytics client is wired in. */ diff --git a/tools/start.mjs b/tools/start.mjs index ff7ec36ac..4b86ea1ca 100644 --- a/tools/start.mjs +++ b/tools/start.mjs @@ -18,6 +18,12 @@ // so proprietary extensions can live outside this repository: // // npm start --server=puter.com --extensions=../puter-private/gui-extensions +// +// `--smtp` builds and runs the inbound SMTP receiver instead of the backend. +// It is a separate process and exits immediately unless +// `userEmail.localServer` is set: +// +// npm start -- --smtp import { spawn } from 'node:child_process'; import path from 'node:path'; @@ -55,9 +61,19 @@ const run = (cmd, args, opts = {}) => new Promise((resolve, reject) => { const server = getFlag('server'); const extensions = getFlag('extensions'); +const smtp = + process.argv.slice(2).includes('--smtp') || + Boolean(process.env.npm_config_smtp); try { - if ( server ) { + if ( smtp ) { + await run(npm, ['run', 'build:ts']); + await run(process.execPath, [ + '--enable-source-maps', + '-r', './dist/src/backend/telemetry.js', + './dist/src/backend/smtp/index.js', + ]); + } else if ( server ) { const args = ['dev-server.js', `--server=${server}`]; if ( extensions ) args.push(`--extensions=${resolveExtensionPaths(extensions)}`); await run(process.execPath, args, {