self hosted mail (#3932)

* self hosted mail

* update lock

* address daniel review
This commit is contained in:
Neal Shah
2026-09-25 21:48:15 -04:00
committed by GitHub
parent dcb61bc4fd
commit 1fc2eae1ae
27 changed files with 2504 additions and 141 deletions
+5
View File
@@ -16,3 +16,8 @@ MARIADB_PASSWORD=replace-with-strong-password
S3_ACCESS_KEY=puter
S3_SECRET_KEY=replace-with-strong-secret
S3_BUCKET=puter-local
# ── Inbound SMTP (compose profile `smtp`) ------------------------------
# Host port mapped to the receiver's 2525. 25 is what an MX record means;
# many hosting providers block it.
SMTP_PORT=25
+2
View File
@@ -79,6 +79,8 @@ ENV PUTER_CONFIG_PATH=/etc/puter/config.json
ENV NODE_OPTIONS=--enable-source-maps
EXPOSE 4100
# The inbound SMTP receiver, when run from this image with the `smtp` profile.
EXPOSE 2525
USER node
+32
View File
@@ -255,6 +255,38 @@
}
},
// ── Inbound mail (separate process) ─────────────────────────────────
// Mailboxes addressed at one of your own domains. `secret` gates the
// ingress endpoint that files a message into the recipient's ~/.mail.
//
// `localServer` additionally runs an SMTP listener that accepts mail from
// other mail servers and hands each message to that endpoint. It runs as
// its own process - `npm run start:smtp`, or the `smtp` compose profile -
// and stores nothing itself. Without it, nothing delivers to the endpoint
// unless you point something else at it.
//
// The listener is unauthenticated by nature: anything on the internet may
// hand it a message for an address in `localDomains`. It performs no spam,
// SPF, DKIM or DMARC checks. Read the self-hosting guide before exposing
// it.
// "userEmail": {
// "secret": "a long random string",
// "localServer": true,
// "localDomains": ["example.com"],
// // Container-internal by default; publish 25 onto it from outside.
// "localPort": 2525,
// // Defaults to <api_base_url>/email/ingress. The endpoint is served
// // on the `api` subdomain, so when this points somewhere internal,
// // `localIngressHost` has to name that virtual host.
// "localIngressUrl": "http://puter:4100/email/ingress",
// "localIngressHost": "api.example.com",
// "localHostname": "mx.example.com",
// "localMaxRecipients": 50,
// // Concurrent connections. Each message is held in memory while it
// // is delivered, so this also bounds memory use.
// "localMaxClients": 20
// },
// ── Sharing ─────────────────────────────────────────────────────────
// Email a recipient who already has an account about a new share. On
// unless set to false; they opt out via the unsubscribe link, or by
+95
View File
@@ -35,6 +35,12 @@ Optional services (compose profile `ai`, opt-in):
| `puter-ollama` | `ollama/ollama` | Local LLM provider (CPU; GPU passthrough opt-in) |
| `puter-ollama-init` | `ollama/ollama` | One-shot — pulls the default model (`tinyllama`) on first boot |
Optional services (compose profile `smtp`, opt-in):
| Container | Image | Role |
| ------------ | ------------------------ | -------------------------------------------------------- |
| `puter-smtp` | `ghcr.io/heyputer/puter` | Receives mail over SMTP and files it into user mailboxes |
State lives under `./puter/data/<service>/`.
---
@@ -513,6 +519,95 @@ Without `--profile ai`, the `ollama` containers stay down and Puter (with `enabl
For GPU acceleration (NVIDIA), uncomment the `deploy:` block under the `ollama` service in [docker-compose.yml](../docker-compose.yml). Requires `nvidia-container-toolkit` on the host.
## Optional: receiving mail over SMTP
Puter can give each account a mailbox at a domain you control. Two pieces are
involved, and they are configured together under `userEmail`:
- an **ingress endpoint** (`POST /email/ingress`) that files a message into the
recipient's `~/.mail`. It exists whenever `userEmail.secret` is set.
- the **`puter-smtp` receiver**, which accepts mail from other mail servers and
hands each message to that endpoint. It runs as its own process, behind the
`smtp` compose profile.
> **Read this before you expose it.** An inbound mail server is unauthenticated
> by nature: anyone on the internet may hand it a message for any address in
> `localDomains`. Puter performs **no** spam filtering and **no** SPF, DKIM or
> DMARC checks — whatever is delivered is stored in the recipient's mailbox and
> counts against their storage. If that is not acceptable, put a filtering relay
> in front of it and point `localDomains` at that instead.
1. Configure it in `puter/config/config.json`:
```json
"userEmail": {
"secret": "a long random string",
"localServer": true,
"localDomains": ["example.com"]
}
```
`secret` is shared between the two halves; generate it with
`openssl rand -hex 32`. `localDomains` lists the domains this server accepts
mail for — anything else is refused at `RCPT TO`, which is what stops it
being used as an open relay. It has no default, and the receiver refuses to
start without it.
The receiver posts to `<api_base_url>/email/ingress` unless you set
`localIngressUrl`. The endpoint is served on the `api` subdomain, so if you
point it at an internal address to skip the reverse proxy, name that virtual
host too or the request will 404:
```json
"localIngressUrl": "http://puter:4100/email/ingress",
"localIngressHost": "api.example.com"
```
`localIngressHost` defaults to the host of `api_base_url`, which is usually
what you want.
2. Bring it up with the `smtp` profile:
```bash
docker compose --profile smtp up -d
docker compose logs -f puter-smtp
```
3. Point DNS at it:
```dns
mx.example.com. A 203.0.113.10
example.com. MX 10 mx.example.com.
```
Also set a `PTR` record for the address that resolves back to
`mx.example.com` — many senders refuse mail from a server without one.
Inbound TCP port 25 must reach the host. **Most cloud providers block port
25 by default and residential ISPs block it permanently**; if you cannot get
it unblocked, this will not work no matter how it is configured. The
container itself binds 2525 (it runs unprivileged) and compose publishes
`${SMTP_PORT:-25}` onto it, so you can move the host port if you need to.
A message is held in memory while it is being delivered, so peak memory is
bounded by `localMaxClients` (default 20) times the 25 MiB message ceiling.
Lower `localMaxClients` on a small host. There is no queue and no retry
schedule — if delivery fails, the sending server is told to try again later and
retrying is its job.
Mail is stored as `message/rfc822` objects under `~/.mail/objects/<date>/` and
read back through `puter.email` in the SDK.
Every setting is documented in
[config.template.jsonc](../config.template.jsonc). Two deliberate limits, both
in the name of keeping the self-hosted path simple: it does not offer STARTTLS,
so mail arrives in the clear, and it applies no per-sender rate limiting. Put a
filtering relay in front if either matters to you.
Note this is a *receiving* server only: it never sends, and it must not be used
for authenticated submission. Outbound transactional mail is the separate
`email` block above.
## Building from source instead of pulling
To run a local build against the full stack, use a source checkout and complete
+26
View File
@@ -286,6 +286,32 @@ services:
retries: 3
start_period: 30s
# Inbound mail. Behind the `smtp` profile, so it only starts when asked for:
# docker compose --profile smtp up -d
# Accepts SMTP from other mail servers and hands each message to the ingress
# endpoint, which files it into the recipient's mailbox. Needs `userEmail`
# configured — see doc/self-hosting.md#optional-receiving-mail-over-smtp.
puter-smtp:
profiles: ["smtp"]
image: ghcr.io/heyputer/puter:main
pull_policy: always
container_name: puter-smtp
restart: unless-stopped
command: ["node", "-r", "./dist/src/backend/telemetry.js", "./dist/src/backend/smtp/index.js"]
depends_on:
puter:
condition: service_healthy
ports:
# The container runs unprivileged and binds 2525; 25 is what an MX record
# means, and many providers block it.
- "${SMTP_PORT:-25}:2525"
environment:
PUID: 1000
PGID: 1000
volumes:
# The same config.json the backend reads; only `userEmail` applies here.
- ./puter/config:/etc/puter:z
caddy:
image: caddy:2.11-alpine
container_name: puter-caddy
+268
View File
@@ -0,0 +1,268 @@
import { EventEmitter } from 'node:events';
import { Readable } from 'node:stream';
import type { Request, Response } from 'express';
import { beforeEach, describe, expect, test, vi } from 'vitest';
const SECRET = 'ingress-secret';
// Only `extension` is stubbed; the secret comparison and the routing are the
// real code paths, and the fake service/store layer is the boundary.
const state = vi.hoisted(() => ({
config: { userEmail: { secret: 'ingress-secret' } } as Record<
string,
unknown
>,
routes: [] as Array<{
path: string;
handler: (req: unknown, res: unknown) => unknown;
}>,
users: new Map<string, Record<string, unknown>>(),
writes: [] as Array<{
userId: number;
path: string;
homeRegion?: string;
}>,
}));
vi.mock('@heyputer/backend/src/extensions', () => ({
extension: {
get config() {
return state.config;
},
post(
path: string,
_opts: unknown,
handler: (req: unknown, res: unknown) => unknown,
) {
state.routes.push({ path, handler });
},
import(layer: string) {
if (layer === 'store') {
return {
user: {
getByUsername: async (username: string) =>
state.users.get(username) ?? null,
},
};
}
return {
fs: {
async write(
userId: number,
{ fileMetadata }: { fileMetadata: { path: string } },
_uploadTracker?: unknown,
_storageAllowanceMax?: number,
homeRegion?: string,
) {
state.writes.push({
userId,
path: fileMetadata.path,
homeRegion,
});
return { fsEntry: { path: fileMetadata.path } };
},
},
};
},
},
}));
import './index.js';
const permanent = (id: number, username: string) => ({
id,
username,
email: `${username}@example.com`,
password: '$2b$10$hash',
});
/** No password and no email is what makes an account temporary. */
const temp = (id: number, username: string) => ({
id,
username,
email: null,
password: null,
});
const ingress = async (
to: string,
over: { secret?: string; contentLength?: string | null } = {},
) => {
const route = state.routes.find((r) => r.path === '/email/ingress');
if (!route) throw new Error('route not registered');
const body = Buffer.from('From: someone@example.com\r\n\r\nhello\r\n');
const req = Readable.from([body], { objectMode: false }) as unknown as {
headers: Record<string, string>;
query: Record<string, string>;
};
// `contentLength: null` sends no length header at all, which is the one
// framing the HTTP parser does not bound for us.
req.headers =
over.contentLength === null
? {}
: {
'content-length': over.contentLength ?? String(body.length),
};
req.query = { SECRET: over.secret ?? SECRET, to, subject: 'hi' };
let status = 200;
let ended = false;
// `end` takes (chunk, encoding) and no callback, the way the compression
// middleware leaves it, and emits `finish` — which is what the refusal
// paths hang their drain/destroy on. A stub that accepts a callback
// instead hides the bug that took the 413 path to production as a 500.
const emitter = new EventEmitter();
const res = {
status(code: number) {
status = code;
return res;
},
end(chunk?: unknown, encoding?: BufferEncoding) {
if (chunk) Buffer.byteLength(chunk as string, encoding);
ended = true;
emitter.emit('finish');
},
once(event: string, listener: () => void) {
emitter.once(event, listener);
return res;
},
} as unknown as Response;
await route.handler(req as unknown as Request, res);
return {
status,
ended,
drained: (req as unknown as Readable).readableFlowing,
destroyed: (req as unknown as Readable).destroyed,
};
};
beforeEach(() => {
state.users.clear();
state.writes.length = 0;
});
describe('temporary accounts cannot receive', () => {
test('mail for a temp account is accepted and dropped', async () => {
state.users.set('tempy', temp(1, 'tempy'));
const { status, ended, drained } = await ingress('tempy@puter.email');
// Accepted, so the relay neither retries nor bounces to the sender.
expect(status).toBe(200);
expect(ended).toBe(true);
// The body is read off the wire rather than left unconsumed.
expect(drained).toBe(true);
expect(state.writes).toEqual([]);
});
test('mail for a permanent account still lands in the mailbox', async () => {
state.users.set('bob', permanent(2, 'bob'));
const { status } = await ingress('bob@puter.email');
expect(status).toBe(200);
expect(state.writes).toHaveLength(1);
expect(state.writes[0].userId).toBe(2);
expect(state.writes[0].path).toMatch(/^\/bob\/\.mail\/objects\//);
});
test('an unknown address is still rejected outright', async () => {
const { status } = await ingress('nobody@puter.email');
expect(status).toBe(404);
expect(state.writes).toEqual([]);
});
});
describe('inbound mail is placed in the recipient home region', () => {
/** A permanent account with explicit placement columns. */
const placed = (
id: number,
username: string,
over: { home?: string | null; signup_server?: string | null },
) => ({
...permanent(id, username),
home: null,
signup_server: null,
...over,
});
test("uses the recipient's own home region when they have one", async () => {
state.users.set('bob', placed(2, 'bob', { home: 'frankfurt' }));
await ingress('bob@puter.email');
expect(state.writes[0].homeRegion).toBe('frankfurt');
});
test('falls back to the server that served their signup', async () => {
// Every account that predates the `home` column is this case.
state.users.set(
'bob',
placed(2, 'bob', { home: null, signup_server: 'london' }),
);
await ingress('bob@puter.email');
expect(state.writes[0].homeRegion).toBe('london');
});
test('falls back to the primary region when neither is set', async () => {
state.users.set('bob', placed(2, 'bob', {}));
await ingress('bob@puter.email');
expect(state.writes[0].homeRegion).toBe('oregon');
});
test('home wins over signup_server when both are set', async () => {
state.users.set(
'bob',
placed(2, 'bob', { home: 'sydney', signup_server: 'london' }),
);
await ingress('bob@puter.email');
expect(state.writes[0].homeRegion).toBe('sydney');
});
});
describe('refusals dispose of the body nobody read', () => {
test('an unknown recipient drains it and keeps the connection', async () => {
// The secret checked out and the length is under the cap, so the body
// was ours to read: discard it rather than resetting a usable socket.
const { status, drained, destroyed } =
await ingress('nobody@puter.email');
expect(status).toBe(404);
expect(drained).toBe(true);
expect(destroyed).toBe(false);
});
test('a bad secret closes rather than reading the upload', async () => {
state.users.set('bob', permanent(2, 'bob'));
const { status, destroyed } = await ingress('bob@puter.email', {
secret: 'not-the-secret',
});
expect(status).toBe(403);
// Nothing authenticated this caller; their bytes are not worth reading.
expect(destroyed).toBe(true);
expect(state.writes).toEqual([]);
});
test('no content-length is refused with 411', async () => {
state.users.set('bob', permanent(2, 'bob'));
const { status, destroyed } = await ingress('bob@puter.email', {
contentLength: null,
});
expect(status).toBe(411);
expect(destroyed).toBe(true);
expect(state.writes).toEqual([]);
});
test('a declared length over the cap is refused with 413', async () => {
state.users.set('bob', permanent(2, 'bob'));
const { status, destroyed } = await ingress('bob@puter.email', {
contentLength: String(25 * 1024 * 1024 + 1),
});
expect(status).toBe(413);
expect(destroyed).toBe(true);
expect(state.writes).toEqual([]);
});
});
+104
View File
@@ -0,0 +1,104 @@
import { extension } from '@heyputer/backend/src/extensions';
import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto';
import {
MAX_MESSAGE_BYTES,
isTempUser,
puterEmailUsername,
storeInboxMessage,
} from '@heyputer/backend/src/services/email/mailbox.js';
// copied form peer secrets checker
const COMPARE_KEY = randomBytes(32);
const secretsEqual = (a: string, b: string): boolean =>
timingSafeEqual(
createHmac('sha256', COMPARE_KEY).update(a).digest(),
createHmac('sha256', COMPARE_KEY).update(b).digest(),
);
const INGRESS_SECRET = (
(extension.config as Record<string, unknown>).userEmail as
| { secret?: string }
| undefined
)?.secret;
if (!INGRESS_SECRET) {
console.warn(
'[useremailrx] config.userEmail.secret unset - email ingress disabled',
);
} else {
extension.post(
'/email/ingress',
{
subdomain: 'api',
},
async (req, res) => {
// Answer first, then deal with the body nobody read. Touching the
// request before the response has flushed takes the response down
// with the connection and the caller sees a reset with no status,
// so this hangs off `finish` — and it has to be the event rather
// than `res.end(cb)`, because the compression middleware replaces
// `res.end` with an `(chunk, encoding)` signature that has no
// callback parameter, so a callback lands in `chunk` and dies in
// `Buffer.byteLength`.
//
// `drain` discards the rest of a body we were entitled to read and
// leaves the connection reusable. `close` is for bodies we do not
// want to read at all: an unauthenticated caller's, or one whose
// declared length is missing or over the cap.
const refuse = (status: number, how: 'drain' | 'close') => {
res.once('finish', () =>
how === 'drain' ? req.resume() : req.destroy(),
);
res.status(status).end();
};
const presented = req.query.SECRET;
if (
typeof presented !== 'string' ||
!secretsEqual(presented, INGRESS_SECRET)
) {
// Nothing has authenticated this caller, so refuse to spend
// bandwidth reading whatever they were uploading.
refuse(403, 'close');
return;
}
const size = Number(req.headers['content-length']);
if (!Number.isInteger(size) || size <= 0) {
refuse(411, 'close');
return;
}
if (size > MAX_MESSAGE_BYTES) {
refuse(413, 'close');
return;
}
const stores = extension.import('store');
const services = extension.import('service');
const user = await stores.user.getByUsername(
puterEmailUsername(req.query.to as string),
);
if (!user) {
refuse(404, 'drain');
return;
}
if (isTempUser(user)) {
req.resume();
res.end();
return;
}
await storeInboxMessage(services.fs, user, {
subject: req.query.subject,
content: req,
size,
});
const { SECRET: _secret, ..._loggableQuery } = req.query;
// console.log('got email: ', puterUser, loggableQuery, size);
res.end();
},
);
}
+12
View File
@@ -0,0 +1,12 @@
{
"name": "emailrx",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "emailrx",
"version": "1.0.0"
}
}
}
+6
View File
@@ -0,0 +1,6 @@
{
"name": "emailrx",
"version": "1.0.0",
"main": "index.ts",
"description": "Internal service which handles routing puter emails to users"
}
+53
View File
@@ -1216,6 +1216,7 @@
},
"node_modules/@clack/prompts/node_modules/is-unicode-supported": {
"version": "1.3.0",
"extraneous": true,
"inBundle": true,
"license": "MIT",
"engines": {
@@ -6153,6 +6154,17 @@
"@types/node": "*"
}
},
"node_modules/@types/smtp-server": {
"version": "3.5.13",
"resolved": "https://registry.npmjs.org/@types/smtp-server/-/smtp-server-3.5.13.tgz",
"integrity": "sha512-S3rGl2KbViH+98/CgHipPIWgtAFnjxLsppxIRbgJHNuZL7Y+py+7kZjT7xS+wOmCxYTn4O7IqLqkvekg99MDVQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*",
"@types/nodemailer": "*"
}
},
"node_modules/@types/tedious": {
"version": "4.0.14",
"resolved": "https://registry.npmjs.org/@types/tedious/-/tedious-4.0.14.tgz",
@@ -10797,6 +10809,12 @@
"node": ">= 10"
}
},
"node_modules/ipv6-normalize": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/ipv6-normalize/-/ipv6-normalize-1.0.1.tgz",
"integrity": "sha512-Bm6H79i01DjgGTCWjUuCjJ6QDo1HB96PT/xCYuyJUP9WFbVDrLSbG4EZCvOCun2rNswZb0c3e4Jt/ws795esHA==",
"license": "MIT"
},
"node_modules/is-binary-path": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
@@ -14488,6 +14506,15 @@
"node": ">=6"
}
},
"node_modules/punycode.js": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz",
"integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==",
"license": "MIT",
"engines": {
"node": ">=6"
}
},
"node_modules/puter-mcp-connector": {
"resolved": "src/mcp-connector",
"link": true
@@ -15530,6 +15557,29 @@
"integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==",
"license": "MIT"
},
"node_modules/smtp-server": {
"version": "3.19.13",
"resolved": "https://registry.npmjs.org/smtp-server/-/smtp-server-3.19.13.tgz",
"integrity": "sha512-W1CjNcrPqW4+1gUOYRyjPvTOxu7GgKR9+7l50yQBF0GM8v8jpYwheHFCHX7s+p7pYI8iXKQQZiF2Rv2Na/3i2g==",
"license": "MIT-0",
"dependencies": {
"ipv6-normalize": "1.0.1",
"nodemailer": "10.0.10",
"punycode.js": "2.3.1"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/smtp-server/node_modules/nodemailer": {
"version": "10.0.10",
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-10.0.10.tgz",
"integrity": "sha512-He9XskOFms62SyAKkLu8CcGcYHAo+BSHSsORI8s4PJH8qZgZY4L4lDfvyN0twvmbpyG+eGrxpyBlskj9d9rJ8A==",
"license": "MIT-0",
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/socket.io": {
"version": "4.8.3",
"resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.8.3.tgz",
@@ -17650,8 +17700,10 @@
"openai": "^6.34.0",
"otpauth": "^9.5.2",
"pg": "^8.21.0",
"postal-mime": "^3.0.0",
"replicate": "^1.0.0",
"sharp": "^0.35.4",
"smtp-server": "^3.19.13",
"socket.io": "^4.8.3",
"svg-captcha": "^1.4.0",
"together-ai": "^0.33.0",
@@ -17670,6 +17722,7 @@
"@types/node": "^24.13.6",
"@types/nodemailer": "^8.0.1",
"@types/pg": "^8.6.1",
"@types/smtp-server": "^3.5.13",
"@types/validator": "^13.15.10",
"pgmock": "^1.0.3",
"typescript": "^5.9.3",
+1
View File
@@ -50,6 +50,7 @@
"build:workerLib:coverage": "cd src/puter-js && npm run build:coverage && cd ../worker && npm run build",
"start:gui": "nodemon --exec \"node dev-server.js\" ",
"start": "node ./tools/start.mjs",
"start:smtp": "node ./tools/start.mjs --smtp",
"dev": "npm start",
"build": "npm run setupExtensions && npm run build:ts && cd src/gui && node ./build.js && cd ../puter-js && npm run build",
"build:workerLib": "cd src/puter-js && npm run build && cd ../worker && npm run build",
+159
View File
@@ -0,0 +1,159 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { existsSync, readFileSync } from 'node:fs';
import path from 'node:path';
import type { IConfig } from './types';
// Config resolution order:
// 1. `process.env.PUTER_CONFIG_PATH` — absolute path to a config file. Used
// by deployments whose outer bootstrap writes a merged config to a known
// location.
// 2. `<PACKAGE_ROOT>/config.json` — user's runtime override (gitignored),
// deep-merged over config.default.json so users can omit keys they
// don't care to override (e.g. gui_assets_root, database).
// 3. `<PACKAGE_ROOT>/config.default.json` — bundled OSS defaults.
//
// Post-flatten depth: compiled file is at `packages/puter/dist/src/backend/index.js`,
// so three `..`s land at `packages/puter/`.
const PACKAGE_ROOT = path.resolve(__dirname, '../../..');
// Root of the running code tree. Matches PACKAGE_ROOT for a source run, but
// points at `dist/` for a compiled run — so config-declared paths like
// `./extensions` resolve to `dist/extensions` at runtime without the config
// having to know about the build layout.
const RUNTIME_ROOT = path.resolve(__dirname, '../..');
const isPlainObject = (v: unknown): v is Record<string, unknown> =>
typeof v === 'object' && v !== null && !Array.isArray(v);
const deepMerge = <T extends Record<string, unknown>>(
base: T,
override: Record<string, unknown>,
): T => {
const out: Record<string, unknown> = { ...base };
for (const [k, v] of Object.entries(override)) {
out[k] =
isPlainObject(v) && isPlainObject(out[k])
? deepMerge(out[k] as Record<string, unknown>, v)
: v;
}
return out as T;
};
export const loadConfig = (): IConfig => {
const envPath = process.env.PUTER_CONFIG_PATH;
const runtimePath = path.join(PACKAGE_ROOT, 'config.json');
const defaultPath = path.join(PACKAGE_ROOT, 'config.default.json');
const defaults = existsSync(defaultPath)
? (JSON.parse(readFileSync(defaultPath, 'utf8')) as Record<
string,
unknown
>)
: {};
// Runtime override path: env wins, then config.json, else no override
// (we still return defaults so single-file installs work).
const overridePath =
envPath && existsSync(envPath)
? envPath
: existsSync(runtimePath)
? runtimePath
: null;
console.log(`[config] defaults from ${defaultPath}`);
if (overridePath) console.log(`[config] override from ${overridePath}`);
const override = overridePath
? (JSON.parse(readFileSync(overridePath, 'utf8')) as Record<
string,
unknown
>)
: {};
const config = deepMerge(defaults, override) as IConfig;
if (!config.version) {
const pkgPath = path.join(PACKAGE_ROOT, 'package.json');
if (existsSync(pkgPath)) {
try {
const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')) as {
version?: string;
};
if (pkg.version) config.version = pkg.version;
} catch {
// fall through — /version returns 'unknown'
}
}
}
// Computed defaults. `origin` and `pub_port` are the externally-visible
// URL+port — what the browser sees. Separate from `port`, which is the
// bind port (can differ when behind a reverse proxy). Code paths that
// build self-referential URLs (GUI bootstrap, email links, OIDC callbacks)
// depend on `origin` having the right port baked in.
if (config.pub_port === undefined) config.pub_port = config.port;
const protocol = config.protocol ?? 'http';
const domain = config.domain ?? 'localhost';
const portSuffix =
config.pub_port === 80 || config.pub_port === 443
? ''
: `:${config.pub_port}`;
if (config.origin === undefined) {
config.origin = `${protocol}://${domain}${portSuffix}`;
}
// API lives on the `api.` subdomain on the same host+port as the main
// origin (see PuterRouter subdomain handling in server.ts). Compute it
// from pub_port/domain so a single-port override (e.g. port: 5101) flows
// through to the GUI bootstrap without users having to restate the URL.
if (config.api_base_url === undefined) {
config.api_base_url = `${protocol}://api.${domain}${portSuffix}`;
}
// Resolve path-valued config fields. Two different roots:
// - `extensions` uses RUNTIME_ROOT because extensions ship inside the
// build output (dist/extensions) and the loader's dynamic import()
// resolves relative paths against the *importing* module file.
// - GUI/puter-js/builtin-apps use PACKAGE_ROOT because those assets
// live only in the source tree (not copied into dist/) and are served
// via express.static at runtime.
const resolveRuntime = (p: string): string =>
path.isAbsolute(p) ? p : path.resolve(RUNTIME_ROOT, p);
const resolvePackage = (p: string): string =>
path.isAbsolute(p) ? p : path.resolve(PACKAGE_ROOT, p);
if (Array.isArray(config.extensions)) {
config.extensions = config.extensions.map(resolveRuntime);
}
if (typeof config.gui_assets_root === 'string') {
config.gui_assets_root = resolvePackage(config.gui_assets_root);
}
if (typeof config.puterjs_root === 'string') {
config.puterjs_root = resolvePackage(config.puterjs_root);
}
if (isPlainObject(config.builtin_apps)) {
for (const [k, v] of Object.entries(config.builtin_apps)) {
if (typeof v === 'string') {
(config.builtin_apps as Record<string, string>)[k] =
resolvePackage(v);
}
}
}
return config;
};
+1 -140
View File
@@ -17,160 +17,21 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { existsSync, readFileSync } from 'node:fs';
import path from 'node:path';
import { isSpanContextValid, trace } from '@opentelemetry/api';
import { puterClients } from './clients';
import { loadConfig } from './config';
import { puterControllers } from './controllers';
import { puterDrivers } from './drivers';
import { PuterServer } from './server';
import { puterServices } from './services';
import { puterStores } from './stores';
import type { IConfig } from './types';
import { createGracefulShutdown } from './util/gracefulShutdown.js';
import { installJsonConsole } from './util/jsonConsole.js';
// Config resolution order:
// 1. `process.env.PUTER_CONFIG_PATH` — absolute path to a config file. Used
// by deployments whose outer bootstrap writes a merged config to a known
// location.
// 2. `<PACKAGE_ROOT>/config.json` — user's runtime override (gitignored),
// deep-merged over config.default.json so users can omit keys they
// don't care to override (e.g. gui_assets_root, database).
// 3. `<PACKAGE_ROOT>/config.default.json` — bundled OSS defaults.
//
// Post-flatten depth: compiled file is at `packages/puter/dist/src/backend/index.js`,
// so three `..`s land at `packages/puter/`.
const PACKAGE_ROOT = path.resolve(__dirname, '../../..');
// Root of the running code tree. Matches PACKAGE_ROOT for a source run, but
// points at `dist/` for a compiled run — so config-declared paths like
// `./extensions` resolve to `dist/extensions` at runtime without the config
// having to know about the build layout.
const RUNTIME_ROOT = path.resolve(__dirname, '../..');
// How long a node with a server identity keeps serving in-flight work after
// it stops accepting connections.
const GRACEFUL_DRAIN_MS = 90_000;
const isPlainObject = (v: unknown): v is Record<string, unknown> =>
typeof v === 'object' && v !== null && !Array.isArray(v);
const deepMerge = <T extends Record<string, unknown>>(
base: T,
override: Record<string, unknown>,
): T => {
const out: Record<string, unknown> = { ...base };
for (const [k, v] of Object.entries(override)) {
out[k] =
isPlainObject(v) && isPlainObject(out[k])
? deepMerge(out[k] as Record<string, unknown>, v)
: v;
}
return out as T;
};
const loadConfig = (): IConfig => {
const envPath = process.env.PUTER_CONFIG_PATH;
const runtimePath = path.join(PACKAGE_ROOT, 'config.json');
const defaultPath = path.join(PACKAGE_ROOT, 'config.default.json');
const defaults = existsSync(defaultPath)
? (JSON.parse(readFileSync(defaultPath, 'utf8')) as Record<
string,
unknown
>)
: {};
// Runtime override path: env wins, then config.json, else no override
// (we still return defaults so single-file installs work).
const overridePath =
envPath && existsSync(envPath)
? envPath
: existsSync(runtimePath)
? runtimePath
: null;
console.log(`[config] defaults from ${defaultPath}`);
if (overridePath) console.log(`[config] override from ${overridePath}`);
const override = overridePath
? (JSON.parse(readFileSync(overridePath, 'utf8')) as Record<
string,
unknown
>)
: {};
const config = deepMerge(defaults, override) as IConfig;
if (!config.version) {
const pkgPath = path.join(PACKAGE_ROOT, 'package.json');
if (existsSync(pkgPath)) {
try {
const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')) as {
version?: string;
};
if (pkg.version) config.version = pkg.version;
} catch {
// fall through — /version returns 'unknown'
}
}
}
// Computed defaults. `origin` and `pub_port` are the externally-visible
// URL+port — what the browser sees. Separate from `port`, which is the
// bind port (can differ when behind a reverse proxy). Code paths that
// build self-referential URLs (GUI bootstrap, email links, OIDC callbacks)
// depend on `origin` having the right port baked in.
if (config.pub_port === undefined) config.pub_port = config.port;
const protocol = config.protocol ?? 'http';
const domain = config.domain ?? 'localhost';
const portSuffix =
config.pub_port === 80 || config.pub_port === 443
? ''
: `:${config.pub_port}`;
if (config.origin === undefined) {
config.origin = `${protocol}://${domain}${portSuffix}`;
}
// API lives on the `api.` subdomain on the same host+port as the main
// origin (see PuterRouter subdomain handling in server.ts). Compute it
// from pub_port/domain so a single-port override (e.g. port: 5101) flows
// through to the GUI bootstrap without users having to restate the URL.
if (config.api_base_url === undefined) {
config.api_base_url = `${protocol}://api.${domain}${portSuffix}`;
}
// Resolve path-valued config fields. Two different roots:
// - `extensions` uses RUNTIME_ROOT because extensions ship inside the
// build output (dist/extensions) and the loader's dynamic import()
// resolves relative paths against the *importing* module file.
// - GUI/puter-js/builtin-apps use PACKAGE_ROOT because those assets
// live only in the source tree (not copied into dist/) and are served
// via express.static at runtime.
const resolveRuntime = (p: string): string =>
path.isAbsolute(p) ? p : path.resolve(RUNTIME_ROOT, p);
const resolvePackage = (p: string): string =>
path.isAbsolute(p) ? p : path.resolve(PACKAGE_ROOT, p);
if (Array.isArray(config.extensions)) {
config.extensions = config.extensions.map(resolveRuntime);
}
if (typeof config.gui_assets_root === 'string') {
config.gui_assets_root = resolvePackage(config.gui_assets_root);
}
if (typeof config.puterjs_root === 'string') {
config.puterjs_root = resolvePackage(config.puterjs_root);
}
if (isPlainObject(config.builtin_apps)) {
for (const [k, v] of Object.entries(config.builtin_apps)) {
if (typeof v === 'string') {
(config.builtin_apps as Record<string, string>)[k] =
resolvePackage(v);
}
}
}
return config;
};
// if called directly, start the server
if (require.main === module) {
const config = loadConfig();
+3
View File
@@ -57,8 +57,10 @@
"openai": "^6.34.0",
"otpauth": "^9.5.2",
"pg": "^8.21.0",
"postal-mime": "^3.0.0",
"replicate": "^1.0.0",
"sharp": "^0.35.4",
"smtp-server": "^3.19.13",
"socket.io": "^4.8.3",
"svg-captcha": "^1.4.0",
"together-ai": "^0.33.0",
@@ -77,6 +79,7 @@
"@types/node": "^24.13.6",
"@types/nodemailer": "^8.0.1",
"@types/pg": "^8.6.1",
"@types/smtp-server": "^3.5.13",
"@types/validator": "^13.15.10",
"pgmock": "^1.0.3",
"typescript": "^5.9.3",
+223
View File
@@ -0,0 +1,223 @@
import { Readable } from 'node:stream';
import { v4 as uuidv4 } from 'uuid';
import { afterAll, beforeAll, describe, expect, test } from 'vitest';
import type { PuterServer } from '../../server.js';
import type { UserRow } from '../../stores/user/UserStore.js';
import { createTestUser, setupTestServer } from '../../testUtil.js';
import {
MAIL_CONTENT_TYPE,
copyIntoInbox,
findMailboxOwner,
hasMailbox,
isPuterEmailAddress,
isTempUser,
mailDay,
mailFolderPath,
mailObjectName,
mailboxPath,
puterEmailAddressesOf,
puterEmailUsername,
storeInboxMessage,
} from './mailbox.js';
const UUID_V7 =
/^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
// A real in-memory backend rather than stand-in services: these functions
// hand their arguments straight to the filesystem, so a fake would only prove
// which methods were called.
let server: PuterServer;
let dan: UserRow;
// `frankfurt` is wired to the same bucket the test server already has, so a
// message filed for a Frankfurt account lands somewhere real and the region it
// was placed in can be read back off the entry.
const FRANKFURT_REGION = 'eu-central-1';
beforeAll(async () => {
server = await setupTestServer({
servers: {
frankfurt: {
bucket: 'puter-local',
bucketRegion: FRANKFURT_REGION,
},
},
} as never);
await createTestUser(server, { username: 'dan', password: 'secret123!' });
dan = (await server.stores.user.getByUsername('dan')) as UserRow;
});
afterAll(async () => {
await server?.shutdown();
});
describe('addresses', () => {
test('both Puter domains are internal, case-insensitively', () => {
expect(isPuterEmailAddress('dan@puter.email')).toBe(true);
expect(isPuterEmailAddress('Dan@PuterStaging.Email')).toBe(true);
expect(isPuterEmailAddress('dan@apps.puter.email')).toBe(false);
expect(isPuterEmailAddress('dan@example.com')).toBe(false);
expect(isPuterEmailAddress('not-an-address')).toBe(false);
});
test('the local part is the username', () => {
expect(puterEmailUsername('dan@puter.email')).toBe('dan');
});
test('an account answers at every Puter domain', () => {
expect(puterEmailAddressesOf('dan')).toEqual([
'dan@puter.email',
'dan@puterstaging.email',
]);
});
});
describe('account eligibility', () => {
test('temp means no password and no email; either alone is not temp', () => {
expect(isTempUser({ password: null, email: null })).toBe(true);
expect(isTempUser({ password: null, email: 'a@b.c' })).toBe(false);
expect(isTempUser({ password: 'x', email: null })).toBe(false);
});
test('the owner of an address is its permanent account, else nothing', async () => {
// A temp account has neither password nor email, which is how one is
// minted before signup completes.
await server.stores.user.create({
username: 'tmp',
uuid: uuidv4(),
password: null,
email: null,
requires_email_confirmation: false,
});
const owner = await findMailboxOwner(
server.stores.user,
'dan@puter.email',
);
expect(owner?.username).toBe('dan');
expect(
await findMailboxOwner(server.stores.user, 'tmp@puter.email'),
).toBeNull();
expect(
await findMailboxOwner(server.stores.user, 'nobody@puter.email'),
).toBeNull();
});
test('a mailbox is set up when ~/.mail is a directory', async () => {
expect(await hasMailbox(server.stores.fsEntry, dan)).toBe(false);
await server.services.fs.mkdir(dan.id, {
path: mailboxPath(dan.username),
createMissingParents: true,
});
expect(await hasMailbox(server.stores.fsEntry, dan)).toBe(true);
});
});
describe('layout', () => {
test('folders sit under ~/.mail by UTC day', () => {
expect(mailboxPath('dan')).toBe('/dan/.mail');
expect(mailFolderPath('dan', 'objects', '2026-09-15')).toBe(
'/dan/.mail/objects/2026-09-15',
);
expect(mailDay(new Date('2026-09-15T23:59:59.000Z'))).toBe(
'2026-09-15',
);
});
test('an object name is a v7 uuid and the base64url subject', () => {
const [id, encoded] = mailObjectName('Hello, wörld').split('--');
expect(id).toMatch(UUID_V7);
expect(Buffer.from(encoded, 'base64url').toString('utf8')).toBe(
'Hello, wörld',
);
});
test('only the first 80 subject characters survive into the name', () => {
const encoded = mailObjectName('x'.repeat(200)).split('--')[1];
expect(Buffer.from(encoded, 'base64url').toString('utf8')).toBe(
'x'.repeat(80),
);
});
test('a missing or non-string subject encodes as empty', () => {
expect(mailObjectName(undefined)).toMatch(/--$/);
expect(mailObjectName(['a'])).toMatch(/--$/);
});
});
describe('storeInboxMessage', () => {
test("files under today's inbox folder, in the owner's home region", async () => {
await createTestUser(server, {
username: 'frank',
password: 'secret123!',
});
const created = (await server.stores.user.getByUsername(
'frank',
)) as UserRow;
await server.stores.user.update(created.id, { home: 'frankfurt' });
const owner = (await server.stores.user.getByUsername(
'frank',
)) as UserRow;
const entry = await storeInboxMessage(server.services.fs, owner, {
subject: 'hi',
content: Readable.from([Buffer.from('raw')]),
size: 3,
});
expect(entry.path).toBe(
`${mailFolderPath('frank', 'objects', mailDay())}/${entry.name}`,
);
expect(entry.name).toMatch(
new RegExp(`^${UUID_V7.source.slice(1, -1)}--aGk$`),
);
expect(entry.size).toBe(3);
expect(JSON.parse(entry.metadata as string)).toMatchObject({
contentType: MAIL_CONTENT_TYPE,
});
// The message follows the account, rather than landing wherever the
// request happened to arrive.
expect(entry.bucketRegion).toBe(FRANKFURT_REGION);
const stored = await server.stores.fsEntry.getEntryByPath(entry.path);
expect(stored?.uuid).toBe(entry.uuid);
});
test('an explicit day and name are used as given', async () => {
const entry = await storeInboxMessage(server.services.fs, dan, {
content: Buffer.from('raw'),
size: 3,
day: '2020-01-01',
name: 'fixed',
});
expect(entry.path).toBe('/dan/.mail/objects/2020-01-01/fixed');
});
});
describe('copyIntoInbox', () => {
test("makes the day folder, then copies under the caller's name", async () => {
const source = await storeInboxMessage(server.services.fs, dan, {
content: Buffer.from('original'),
size: 8,
day: '2020-02-02',
name: 'source',
});
const copied = await copyIntoInbox(server.services.fs, dan, {
source,
name: 'copied',
day: '2020-03-03',
});
expect(copied.path).toBe('/dan/.mail/objects/2020-03-03/copied');
// The day folder did not exist before the copy; `copy` cannot create
// it, so the function has to.
expect(
await server.stores.fsEntry.getEntryByPath(
'/dan/.mail/objects/2020-03-03',
),
).toBeTruthy();
expect(copied.size).toBe(source.size);
});
});
+186
View File
@@ -0,0 +1,186 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import type { Readable } from 'node:stream';
import type { FSService } from '../fs/FSService.js';
import { resolveHomeRegion } from '../fs/homeRegion.js';
import type { FSEntry } from '../../stores/fs/FSEntry.js';
import type { FSEntryStore } from '../../stores/fs/FSEntryStore.js';
import type { UserRow, UserStore } from '../../stores/user/UserStore.js';
import * as uuid from 'uuid';
/**
* The user mailbox, shared by everything that fills one: external ingress,
* user-to-user sends and transactional sends.
*
* `<username>@<puter domain>` names an account, and that account's mail lives
* under `/<username>/.mail` as `message/rfc822` objects - one folder per UTC
* day, each object named `<uuidv7>--<base64url(subject)>` so a listing can sort
* by time and show the subject without opening anything. The SDK's mailbox
* reader is written against this layout, so it is defined once here.
*/
export const PUTER_EMAIL_DOMAINS = [
'puter.email',
'puterstaging.email',
] as const;
export const MAIL_CONTENT_TYPE = 'message/rfc822';
/** Ceiling on one stored message, inbound or outbound. */
export const MAX_MESSAGE_BYTES = 25 * 1024 * 1024;
export type MailFolder = 'objects' | 'sent' | 'outbox';
const SUBJECT_NAME_CHARS = 80;
export const isPuterEmailAddress = (address: string): boolean =>
(PUTER_EMAIL_DOMAINS as readonly string[]).includes(
address.split('@')[1]?.toLowerCase() ?? '',
);
/** The username a Puter address names. The domain is not checked here. */
export const puterEmailUsername = (address: string): string =>
address.split('@')[0];
/** Every address that resolves to `username`. */
export const puterEmailAddressesOf = (username: string): string[] =>
PUTER_EMAIL_DOMAINS.map((domain) => `${username}@${domain}`);
/**
* A temp account has neither a password nor an email - the same test
* `userProtected` and AuthController use. Read it off a stored user row, never
* off `actor.user`: the actor's copy has `password` stripped, so the check
* there would pass every account.
*/
export const isTempUser = (user: { password?: unknown; email?: unknown }) =>
!user.password && !user.email;
export const mailboxPath = (username: string): string => `/${username}/.mail`;
export const mailFolderPath = (
username: string,
folder: MailFolder,
day: string,
): string => `${mailboxPath(username)}/${folder}/${day}`;
/** `YYYY-MM-DD` in UTC: the day folder a message is filed under. */
export const mailDay = (date: Date = new Date()): string =>
date.toISOString().slice(0, 10);
/** Object name for one message; only the start of the subject survives. */
export const mailObjectName = (subject: unknown): string => {
const text = typeof subject === 'string' ? subject : '';
const encoded = Buffer.from(
text.slice(0, SUBJECT_NAME_CHARS),
'utf8',
).toString('base64url');
return `${uuid.v7()}--${encoded}`;
};
/**
* The account behind a Puter address, or null when nothing can receive there:
* no such user, or a temp account (unverified and free to mint, so it has no
* mailbox in either direction). Callers report both as one thing, so mailing an
* address never tells one user which accounts are temp.
*/
export async function findMailboxOwner(
users: UserStore,
address: string,
): Promise<UserRow | null> {
const user = await users.getByUsername(puterEmailUsername(address));
return user && !isTempUser(user) ? user : null;
}
/** Whether the account has set its mailbox up: `~/.mail` exists. */
export async function hasMailbox(
fsEntries: FSEntryStore,
owner: UserRow,
): Promise<boolean> {
const entry = await fsEntries.getEntryByPath(mailboxPath(owner.username));
return entry?.isDir === true;
}
export interface InboxMessage {
/** Header subject, for the object name. */
subject?: unknown;
content: Buffer | Readable;
/** Exact byte count; storage needs it before the first byte. */
size: number;
/** Defaults to today. */
day?: string;
/** Defaults to a fresh name from `subject`. */
name?: string;
}
/**
* File one message in the owner's inbox. Written into the owner's home region
* rather than wherever the request landed, so the mailbox reads locally.
*/
export async function storeInboxMessage(
fs: FSService,
owner: UserRow,
message: InboxMessage,
): Promise<FSEntry> {
const day = message.day ?? mailDay();
const name = message.name ?? mailObjectName(message.subject);
const { fsEntry } = await fs.write(
owner.id,
{
fileMetadata: {
path: `${mailFolderPath(owner.username, 'objects', day)}/${name}`,
size: message.size,
contentType: MAIL_CONTENT_TYPE,
createMissingParents: true,
overwrite: false,
dedupeName: false,
},
fileContent: message.content,
},
undefined,
undefined,
resolveHomeRegion(owner),
);
return fsEntry;
}
/**
* File an already-stored message in the owner's inbox by copying it, so the
* bytes are never read back. `copy` has no `createMissingParents`, so the day
* folder is made first; `mkdir` is idempotent.
*/
export async function copyIntoInbox(
fs: FSService,
owner: UserRow,
message: { source: FSEntry; name: string; day?: string },
): Promise<FSEntry> {
const parent = await fs.mkdir(owner.id, {
path: mailFolderPath(
owner.username,
'objects',
message.day ?? mailDay(),
),
createMissingParents: true,
});
return await fs.copy(owner.id, {
source: message.source,
destinationParent: parent,
newName: message.name,
});
}
+274
View File
@@ -0,0 +1,274 @@
import http from 'node:http';
import type { AddressInfo } from 'node:net';
import SMTPConnection from 'nodemailer/lib/smtp-connection/index.js';
import { afterAll, afterEach, beforeAll, describe, expect, test } from 'vitest';
import type { ResolvedSmtpConfig } from './config.js';
import { SmtpReceiver } from './SmtpReceiver.js';
interface IngressCall {
to: string | null;
subject: string | null;
from: string | null;
messageid: string | null;
secret: string | null;
contentLength: string | undefined;
contentType: string | undefined;
body: string;
}
let ingress: http.Server;
let ingressUrl: string;
let calls: IngressCall[] = [];
let statusFor: (to: string) => number = () => 200;
let receiver: SmtpReceiver;
let smtpPort: number;
const baseConfig = (
over: Partial<ResolvedSmtpConfig> = {},
): ResolvedSmtpConfig => ({
host: '127.0.0.1',
port: 0,
hostname: 'mx.example.com',
domains: ['example.com'],
ingressUrl,
ingressHost: null,
secret: 'ingress-secret',
maxMessageBytes: 2048,
maxRecipients: 3,
maxClients: 20,
...over,
});
const startReceiver = async (over: Partial<ResolvedSmtpConfig> = {}) => {
receiver = new SmtpReceiver(baseConfig(over));
smtpPort = (await receiver.listen()).port;
};
/** Deliver one message and resolve with the SMTP reply code. */
const deliver = async (opts: {
to: string[];
message?: string;
}): Promise<{ code: number; message: string }> => {
const conn = new SMTPConnection({
host: '127.0.0.1',
port: smtpPort,
secure: false,
ignoreTLS: true,
tls: { rejectUnauthorized: false },
});
return await new Promise((resolve, reject) => {
conn.on('error', reject);
conn.connect(() => {
conn.send(
{
from: 'sender@elsewhere.test',
to: opts.to,
},
opts.message ??
'Subject: Hello\r\nFrom: sender@elsewhere.test\r\n\r\nbody\r\n',
(err: (Error & { responseCode?: number }) | null) => {
conn.close();
if (err) {
resolve({
code: err.responseCode ?? 0,
message: err.message,
});
return;
}
resolve({ code: 250, message: 'ok' });
},
);
});
});
};
/**
* Drive the server with raw SMTP. nodemailer refuses an oversize declaration
* client-side, which would never reach the server being tested.
*/
const rawExchange = async (commands: string[]): Promise<string[]> => {
const { createConnection } = await import('node:net');
return await new Promise((resolve, reject) => {
const replies: string[] = [];
const socket = createConnection(smtpPort, '127.0.0.1');
let next = 0;
socket.setEncoding('utf8');
socket.on('data', (chunk: string) => {
replies.push(chunk.trim());
if (next < commands.length) {
socket.write(`${commands[next++]}\r\n`);
} else {
socket.end();
}
});
socket.on('error', reject);
socket.on('close', () => resolve(replies));
});
};
beforeAll(async () => {
ingress = http.createServer((req, res) => {
const url = new URL(req.url ?? '', 'http://ingress.test');
const chunks: Buffer[] = [];
req.on('data', (c) => chunks.push(c as Buffer));
req.on('end', () => {
const to = url.searchParams.get('to') ?? '';
calls.push({
to,
subject: url.searchParams.get('subject'),
from: url.searchParams.get('from'),
messageid: url.searchParams.get('messageid'),
secret: url.searchParams.get('SECRET'),
contentLength: req.headers['content-length'],
contentType: req.headers['content-type'],
body: Buffer.concat(chunks).toString(),
});
res.statusCode = statusFor(to);
res.end();
});
});
await new Promise<void>((r) => ingress.listen(0, '127.0.0.1', r));
ingressUrl = `http://127.0.0.1:${(ingress.address() as AddressInfo).port}/email/ingress`;
await startReceiver();
});
afterAll(async () => {
await receiver.close();
await new Promise<void>((r) => ingress.close(() => r()));
});
afterEach(() => {
calls = [];
statusFor = () => 200;
});
describe('accepting mail', () => {
test('delivers a message and reports 250', async () => {
const reply = await deliver({ to: ['dan@example.com'] });
expect(reply.code).toBe(250);
expect(calls).toHaveLength(1);
expect(calls[0].to).toBe('dan@example.com');
expect(calls[0].subject).toBe('Hello');
expect(calls[0].from).toBe('sender@elsewhere.test');
expect(calls[0].secret).toBe('ingress-secret');
expect(calls[0].contentType).toBe('message/rfc822');
expect(calls[0].contentLength).toBe(
String(Buffer.byteLength(calls[0].body)),
);
});
test('sends NO-ID when the message has no Message-ID', async () => {
await deliver({ to: ['dan@example.com'] });
expect(calls[0].messageid).toBe('NO-ID');
});
test('posts once per envelope recipient with identical bytes', async () => {
const reply = await deliver({
to: ['dan@example.com', 'sam@example.com'],
});
expect(reply.code).toBe(250);
expect(calls.map((c) => c.to).sort()).toEqual([
'dan@example.com',
'sam@example.com',
]);
expect(calls[0].body).toBe(calls[1].body);
});
test('delivers to the envelope recipient, not the To: header', async () => {
// A blind-copied recipient never appears in the To: header, so reading
// the header instead of the envelope misdelivers the message.
await deliver({
to: ['hidden@example.com'],
message:
'Subject: Bcc test\r\nTo: someone-else@example.com\r\n\r\nbody\r\n',
});
expect(calls).toHaveLength(1);
expect(calls[0].to).toBe('hidden@example.com');
});
test('delivers one copy when an address is repeated', async () => {
await deliver({ to: ['dan@example.com', 'dan@example.com'] });
expect(calls).toHaveLength(1);
});
});
describe('refusing recipients', () => {
test('refuses a domain it does not accept, and posts nothing', async () => {
const reply = await deliver({ to: ['someone@elsewhere.test'] });
expect(reply.code).toBe(550);
expect(reply.message).toContain('Relay access denied');
expect(calls).toHaveLength(0);
});
test('refuses more recipients than the cap allows', async () => {
const reply = await deliver({
to: [
'a@example.com',
'b@example.com',
'c@example.com',
'd@example.com',
],
});
// The first three are accepted, so the message still goes through.
expect(reply.code).toBe(250);
expect(calls).toHaveLength(3);
});
});
describe('message size', () => {
test('refuses before the body when the sender declares an oversize', async () => {
const replies = await rawExchange([
'EHLO test.local',
'MAIL FROM:<sender@elsewhere.test> SIZE=99999999',
'QUIT',
]);
expect(replies.join('\n')).toMatch(/^552/m);
expect(calls).toHaveLength(0);
});
test('advertises the SIZE limit so senders can check before sending', async () => {
const replies = await rawExchange(['EHLO test.local', 'QUIT']);
expect(replies.join('\n')).toContain('SIZE 2048');
});
test('refuses an oversize body that was never declared', async () => {
const big = `Subject: big\r\n\r\n${'x'.repeat(4096)}\r\n`;
const reply = await deliver({ to: ['dan@example.com'], message: big });
expect(reply.code).toBe(552);
expect(calls).toHaveLength(0);
});
});
describe('reporting what the ingress endpoint said', () => {
test('an unknown user becomes a permanent 550', async () => {
statusFor = () => 404;
const reply = await deliver({ to: ['nobody@example.com'] });
expect(reply.code).toBe(550);
});
test('a rejected secret becomes a temporary 451', async () => {
statusFor = () => 403;
const reply = await deliver({ to: ['dan@example.com'] });
expect(reply.code).toBe(451);
});
test('an unreachable endpoint becomes a temporary 451', async () => {
await receiver.close();
await startReceiver({ ingressUrl: 'http://127.0.0.1:1/email/ingress' });
const reply = await deliver({ to: ['dan@example.com'] });
expect(reply.code).toBe(451);
await receiver.close();
await startReceiver();
});
test('reports the first failure when recipients differ', async () => {
// One reply covers the whole message; the sender is told about the
// first address that could not be delivered to.
statusFor = (to) => (to.startsWith('nobody') ? 404 : 200);
const reply = await deliver({
to: ['dan@example.com', 'nobody@example.com'],
});
expect(reply.code).toBe(550);
});
});
+168
View File
@@ -0,0 +1,168 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { SMTPServer } from 'smtp-server';
import type { SMTPServerSession } from 'smtp-server';
import type { ResolvedSmtpConfig } from './config.js';
import {
ACCEPTED,
type SmtpReply,
deliver,
describeTarget,
readHeaderFields,
} from './ingressClient.js';
import { acceptedRecipients } from './recipients.js';
const SOCKET_TIMEOUT_MS = 60_000;
const INGRESS_TIMEOUT_MS = 120_000;
const replyError = (reply: SmtpReply): Error =>
Object.assign(new Error(reply.text), { responseCode: reply.code });
/**
* An SMTP listener that hands each message to the mail ingress endpoint over
* HTTP. It stores nothing and never relays: the only addresses it accepts are
* in a configured domain, and the only place a message goes is that endpoint.
*
* Messages are held in memory while they are delivered, which is why
* `maxClients` matters - it bounds concurrent messages, and so memory, at
* roughly `maxClients` x the 25MiB message ceiling.
*/
export class SmtpReceiver {
readonly #server: SMTPServer;
constructor(private readonly cfg: ResolvedSmtpConfig) {
this.#server = new SMTPServer({
name: cfg.hostname,
// Advertises SIZE, so an oversized message is refused at MAIL FROM
// rather than after it has crossed the wire.
size: cfg.maxMessageBytes,
// Mail arriving from the internet authenticates nobody; the gate is
// the accepted-domain check below.
disabledCommands: ['AUTH'],
authOptional: true,
hideSTARTTLS: true,
maxClients: cfg.maxClients,
socketTimeout: SOCKET_TIMEOUT_MS,
disableReverseLookup: true,
logger: false,
onRcptTo: (address, session, callback) => {
const [recipient] = acceptedRecipients(
[address.address],
cfg.domains,
);
if (!recipient) {
// Refusing every other domain is what keeps this from being
// an open relay.
callback(
replyError({
code: 550,
text: '5.7.1 Relay access denied',
}),
);
return;
}
if (session.envelope.rcptTo.length >= cfg.maxRecipients) {
callback(
replyError({
code: 452,
text: '4.5.3 Too many recipients',
}),
);
return;
}
callback();
},
onData: (stream, session, callback) => {
this.#handle(stream, session)
.then((reply) =>
callback(reply.code === 250 ? null : replyError(reply)),
)
.catch(() =>
callback(
replyError({
code: 451,
text: '4.3.0 Temporary server error',
}),
),
);
},
});
}
async #handle(
stream: NodeJS.ReadableStream & { sizeExceeded?: boolean },
session: SMTPServerSession,
): Promise<SmtpReply> {
const chunks: Buffer[] = [];
for await (const chunk of stream) chunks.push(chunk as Buffer);
if (stream.sizeExceeded) {
return { code: 552, text: '5.3.4 Message too big for system' };
}
const message = Buffer.concat(chunks);
const fields = await readHeaderFields(message);
// The envelope recipient, not the To: header - a blind-copied address
// never appears in the header at all.
const recipients = acceptedRecipients(
session.envelope.rcptTo.map((r) => r.address),
this.cfg.domains,
);
for (const recipient of recipients) {
const reply = await deliver(message, recipient, fields, {
url: this.cfg.ingressUrl,
host: this.cfg.ingressHost,
secret: this.cfg.secret,
timeoutMs: INGRESS_TIMEOUT_MS,
});
if (reply.code !== 250) {
// One reply covers the whole message, so the first failure is
// what the sender is told.
console.warn('[smtp] delivery refused', {
session: session.id,
recipient,
code: reply.code,
// Never the full URL: the secret is in its query string.
ingress: describeTarget(this.cfg.ingressUrl),
});
return reply;
}
}
return ACCEPTED;
}
async listen(): Promise<{ port: number }> {
await new Promise<void>((resolve, reject) => {
this.#server.once('error', reject);
this.#server.listen(this.cfg.port, this.cfg.host, () => {
this.#server.off('error', reject);
resolve();
});
});
const address = this.#server.server.address();
return {
port: typeof address === 'object' && address ? address.port : 0,
};
}
async close(): Promise<void> {
await new Promise<void>((resolve) => this.#server.close(resolve));
}
}
+155
View File
@@ -0,0 +1,155 @@
import { describe, expect, test } from 'vitest';
import type { IConfig } from '../types';
import {
SmtpConfigError,
isLocalServerEnabled,
resolveSmtpConfig,
} from './config.js';
const config = (userEmail: Record<string, unknown> | undefined): IConfig =>
({
api_base_url: 'https://api.example.com',
userEmail,
}) as unknown as IConfig;
const valid = {
secret: 'ingress-secret',
localServer: true,
localDomains: ['example.com'],
};
describe('deciding whether the receiver runs', () => {
test('runs only when localServer is exactly true', () => {
expect(isLocalServerEnabled(config({ ...valid }))).toBe(true);
});
test.each([
['absent userEmail', undefined],
['no localServer', { secret: 's' }],
['localServer false', { secret: 's', localServer: false }],
['a truthy non-boolean', { secret: 's', localServer: 'yes' }],
])('does not run with %s', (_label, userEmail) => {
expect(
isLocalServerEnabled(config(userEmail as Record<string, unknown>)),
).toBe(false);
});
});
describe('resolving receiver config', () => {
test('derives the ingress url from api_base_url', () => {
expect(resolveSmtpConfig(config(valid)).ingressUrl).toBe(
'https://api.example.com/email/ingress',
);
});
test('an explicit ingress url wins', () => {
const cfg = resolveSmtpConfig(
config({ ...valid, localIngressUrl: 'http://ingress.test/x' }),
);
expect(cfg.ingressUrl).toBe('http://ingress.test/x');
});
test('addresses the api virtual host by default', () => {
// The ingress route is served on the api subdomain, so a request has
// to name that host even when it connects somewhere else.
expect(resolveSmtpConfig(config(valid)).ingressHost).toBe(
'api.example.com',
);
});
test('keeps the api host when the url points at an internal address', () => {
const cfg = resolveSmtpConfig(
config({
...valid,
localIngressUrl: 'http://puter:4100/email/ingress',
}),
);
expect(cfg.ingressUrl).toBe('http://puter:4100/email/ingress');
expect(cfg.ingressHost).toBe('api.example.com');
});
test('an explicit ingress host wins', () => {
const cfg = resolveSmtpConfig(
config({ ...valid, localIngressHost: 'api.internal' }),
);
expect(cfg.ingressHost).toBe('api.internal');
});
test('defaults the port to 2525 so no privileged bind is needed', () => {
expect(resolveSmtpConfig(config(valid)).port).toBe(2525);
});
test('port 25 can be opted into', () => {
expect(
resolveSmtpConfig(config({ ...valid, localPort: 25 })).port,
).toBe(25);
});
test('lowercases accepted domains', () => {
const cfg = resolveSmtpConfig(
config({ ...valid, localDomains: ['Example.COM'] }),
);
expect(cfg.domains).toEqual(['example.com']);
});
test('names the first domain as the greeting hostname', () => {
expect(resolveSmtpConfig(config(valid)).hostname).toBe('example.com');
});
test('applies the documented defaults', () => {
expect(resolveSmtpConfig(config(valid))).toMatchObject({
host: '0.0.0.0',
maxRecipients: 50,
maxClients: 20,
});
});
test('takes the message cap from the mailbox module, not config', () => {
// Sharing the constant is what stops the receiver accepting a message
// the ingress endpoint would then refuse.
const cfg = resolveSmtpConfig(
config({ ...valid, maxMessageBytes: 999 }),
);
expect(cfg.maxMessageBytes).toBe(25 * 1024 * 1024);
});
});
describe('refusing an unusable config', () => {
test('requires a secret', () => {
expect(() =>
resolveSmtpConfig(
config({ localServer: true, localDomains: ['e.test'] }),
),
).toThrow(SmtpConfigError);
});
test('requires at least one domain, so it cannot become an open relay', () => {
expect(() =>
resolveSmtpConfig(config({ secret: 's', localServer: true })),
).toThrow(/localDomains/);
});
test('rejects a non-absolute ingress url', () => {
expect(() =>
resolveSmtpConfig(
config({ ...valid, localIngressUrl: '/email/ingress' }),
),
).toThrow(/absolute URL/);
});
test('rejects an impossible port', () => {
expect(() =>
resolveSmtpConfig(config({ ...valid, localPort: 70000 })),
).toThrow(/localPort/);
});
test('names every problem at once', () => {
try {
resolveSmtpConfig(config({ localServer: true }));
expect.unreachable('should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(SmtpConfigError);
expect((err as SmtpConfigError).problems.length).toBeGreaterThan(1);
}
});
});
+119
View File
@@ -0,0 +1,119 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { MAX_MESSAGE_BYTES } from '../services/email/mailbox.js';
import type { IConfig } from '../types';
export interface ResolvedSmtpConfig {
host: string;
port: number;
hostname: string;
domains: string[];
ingressUrl: string;
/** Host header override; the ingress route is gated on the api subdomain. */
ingressHost: string | null;
secret: string;
maxMessageBytes: number;
maxRecipients: number;
maxClients: number;
}
export class SmtpConfigError extends Error {
constructor(public readonly problems: string[]) {
super(`invalid userEmail receiver config:\n- ${problems.join('\n- ')}`);
this.name = 'SmtpConfigError';
}
}
/** Whether the local receiver is switched on at all. */
export const isLocalServerEnabled = (config: IConfig): boolean =>
config.userEmail?.localServer === true;
/**
* Apply defaults and refuse an unusable configuration, naming every problem at
* once. A listener that starts and then rejects every message is harder to
* diagnose than one that will not start.
*/
export const resolveSmtpConfig = (config: IConfig): ResolvedSmtpConfig => {
const cfg = config.userEmail ?? {};
const problems: string[] = [];
const secret = typeof cfg.secret === 'string' ? cfg.secret : '';
if (!secret) problems.push('userEmail.secret must be set');
const domains = Array.isArray(cfg.localDomains)
? cfg.localDomains
.filter((d): d is string => typeof d === 'string' && d.length > 0)
.map((d) => d.toLowerCase())
: [];
if (domains.length === 0) {
problems.push(
'userEmail.localDomains must list at least one domain to accept mail for',
);
}
const ingressUrl =
cfg.localIngressUrl ||
(config.api_base_url
? `${config.api_base_url.replace(/\/$/, '')}/email/ingress`
: '');
if (!ingressUrl) {
problems.push(
'userEmail.localIngressUrl must be set when api_base_url is not configured',
);
} else {
try {
new URL(ingressUrl);
} catch {
problems.push('userEmail.localIngressUrl must be an absolute URL');
}
}
const port = cfg.localPort ?? 2525;
if (!Number.isInteger(port) || port < 1 || port > 65535) {
problems.push('userEmail.localPort must be a port number');
}
// The ingress route is served on the api subdomain, so a URL pointing at an
// internal address still has to address that virtual host by name.
let ingressHost = cfg.localIngressHost ?? null;
if (!ingressHost && config.api_base_url) {
try {
ingressHost = new URL(config.api_base_url).host;
} catch {
ingressHost = null;
}
}
if (problems.length > 0) throw new SmtpConfigError(problems);
return {
host: cfg.localHost ?? '0.0.0.0',
port,
hostname: cfg.localHostname ?? domains[0],
domains,
ingressUrl,
ingressHost,
secret,
maxMessageBytes: MAX_MESSAGE_BYTES,
maxRecipients: cfg.localMaxRecipients ?? 50,
// Bounds concurrent messages, and so memory, since each is held whole.
maxClients: cfg.localMaxClients ?? 20,
};
};
+72
View File
@@ -0,0 +1,72 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { loadConfig } from '../config';
import { SmtpReceiver } from './SmtpReceiver.js';
import {
SmtpConfigError,
isLocalServerEnabled,
resolveSmtpConfig,
} from './config.js';
import { describeTarget } from './ingressClient.js';
// Its own process rather than part of the server: it binds its own port, and a
// fault here should not take the API down with it.
if (require.main === module) {
const config = loadConfig();
if (!isLocalServerEnabled(config)) {
console.log(
'[smtp] userEmail.localServer is not enabled - nothing to run',
);
process.exit(0);
}
let cfg;
try {
cfg = resolveSmtpConfig(config);
} catch (err) {
if (!(err instanceof SmtpConfigError)) throw err;
console.error(`[smtp] ${err.message}`);
process.exit(1);
}
const receiver = new SmtpReceiver(cfg);
receiver
.listen()
.then(({ port }) => {
console.log(
`[smtp] accepting mail for ${cfg.domains.join(', ')} on ${cfg.host}:${port}`,
);
console.log(
`[smtp] forwarding to ${describeTarget(cfg.ingressUrl)}`,
);
})
.catch((err: Error) => {
console.error(`[smtp] failed to listen: ${err.message}`);
process.exit(1);
});
const shutDown = async () => {
await receiver.close();
process.exit(0);
};
process.on('SIGINT', shutDown);
process.on('SIGTERM', shutDown);
}
+205
View File
@@ -0,0 +1,205 @@
import http from 'node:http';
import type { AddressInfo } from 'node:net';
import { afterAll, afterEach, beforeAll, describe, expect, test } from 'vitest';
import {
buildIngressUrl,
deliver,
describeTarget,
readHeaderFields,
replyForStatus,
} from './ingressClient.js';
const SECRET = 'ingress-secret';
describe('building the request url', () => {
const url = (over: Partial<Parameters<typeof buildIngressUrl>[1]> = {}) =>
buildIngressUrl('https://api.example.com/email/ingress', {
to: 'dan@example.com',
subject: 'Hi & bye',
from: 'sender@elsewhere.test',
messageId: '<abc@x>',
secret: SECRET,
...over,
});
test('carries the metadata the endpoint reads', () => {
const u = url();
expect(u.searchParams.get('to')).toBe('dan@example.com');
expect(u.searchParams.get('subject')).toBe('Hi & bye');
expect(u.searchParams.get('from')).toBe('sender@elsewhere.test');
expect(u.searchParams.get('messageid')).toBe('<abc@x>');
expect(u.searchParams.get('SECRET')).toBe(SECRET);
});
test('encodes a unicode subject', () => {
const u = url({ subject: 'héllo → wörld' });
expect(u.searchParams.get('subject')).toBe('héllo → wörld');
expect(u.toString()).not.toContain('→');
});
});
describe('describing the target for a log', () => {
test('never includes the secret', () => {
const described = describeTarget(
`https://api.example.com/email/ingress?SECRET=${SECRET}`,
);
expect(described).toBe('https://api.example.com/email/ingress');
expect(described).not.toContain(SECRET);
});
test('does not throw on an unusable url', () => {
expect(describeTarget('not a url')).not.toContain('not a url');
});
});
describe('reading header fields', () => {
test('reads subject, from and message id', async () => {
const fields = await readHeaderFields(
Buffer.from(
'Subject: Hello there\r\nFrom: Dan <dan@example.com>\r\nMessage-ID: <abc@x>\r\n\r\nbody',
),
);
expect(fields).toEqual({
subject: 'Hello there',
from: 'dan@example.com',
messageId: '<abc@x>',
});
});
test('decodes an encoded-word subject', () => {
// The subject becomes part of the stored object name, so it has to be
// decoded the same way the hosted receiver's pre-parsed header was.
return expect(
readHeaderFields(
Buffer.from('Subject: =?UTF-8?B?SGVsbG8gd29ybGQ=?=\r\n\r\nx'),
),
).resolves.toMatchObject({ subject: 'Hello world' });
});
test('falls back the way the hosted receiver does', async () => {
expect(await readHeaderFields(Buffer.from('X-Other: v\r\n\r\nx'))).toEqual(
{
subject: '',
from: 'anonymous@example.com',
messageId: 'NO-ID',
},
);
});
});
describe('turning a status into an SMTP reply', () => {
test.each([
[200, 250],
[204, 250],
[404, 550],
[413, 552],
[403, 451],
[411, 451],
[429, 451],
[500, 451],
])('%i becomes %i', (status, code) => {
expect(replyForStatus(status).code).toBe(code);
});
test('a rejected secret is temporary, so mail is held not bounced', () => {
// 403 means this end is misconfigured; bouncing would discard the mail.
expect(replyForStatus(403).code).toBe(451);
});
});
describe('delivering to the endpoint', () => {
let server: http.Server;
let base: string;
let received: Array<{
url: string;
headers: http.IncomingHttpHeaders;
body: Buffer;
}> = [];
let status = 200;
let hang = false;
beforeAll(async () => {
server = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on('data', (c) => chunks.push(c as Buffer));
req.on('end', () => {
received.push({
url: req.url ?? '',
headers: req.headers,
body: Buffer.concat(chunks),
});
if (hang) return;
res.statusCode = status;
res.end();
});
});
await new Promise<void>((r) => server.listen(0, '127.0.0.1', r));
base = `http://127.0.0.1:${(server.address() as AddressInfo).port}/email/ingress`;
});
afterAll(async () => {
await new Promise<void>((r) => server.close(() => r()));
});
afterEach(() => {
received = [];
status = 200;
hang = false;
});
const fields = {
subject: 'Hi',
from: 'sender@elsewhere.test',
messageId: '<m@x>',
};
const send = (target: Partial<{ url: string; host: string | null }> = {}) =>
deliver(Buffer.from('Subject: Hi\r\n\r\nthe body'), 'dan@example.com', fields, {
url: base,
secret: SECRET,
timeoutMs: 2000,
...target,
});
test('posts the message with an exact content length', async () => {
const reply = await send();
expect(reply.code).toBe(250);
expect(received).toHaveLength(1);
expect(received[0].headers['content-type']).toBe('message/rfc822');
expect(received[0].headers['content-length']).toBe(
String(received[0].body.byteLength),
);
expect(received[0].body.toString()).toBe('Subject: Hi\r\n\r\nthe body');
expect(received[0].url).toContain('to=dan%40example.com');
});
test('sends the configured host so the api vhost is addressed', async () => {
await send({ host: 'api.example.com' });
expect(received[0].headers.host).toBe('api.example.com');
});
test('falls back to the url host when none is configured', async () => {
await send();
expect(received[0].headers.host).toContain('127.0.0.1');
});
test('maps a refusal from the endpoint', async () => {
status = 404;
expect((await send()).code).toBe(550);
});
test('treats an unreachable endpoint as temporary', async () => {
const reply = await send({ url: 'http://127.0.0.1:1/email/ingress' });
expect(reply.code).toBe(451);
});
test('treats a hung endpoint as temporary', async () => {
hang = true;
const reply = await deliver(
Buffer.from('x'),
'dan@example.com',
fields,
{ url: base, secret: SECRET, timeoutMs: 150 },
);
expect(reply.code).toBe(451);
});
});
+160
View File
@@ -0,0 +1,160 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import http from 'node:http';
import https from 'node:https';
import PostalMime from 'postal-mime';
import { MAIL_CONTENT_TYPE } from '../services/email/mailbox.js';
/**
* An SMTP reply: 250 when the endpoint took the message, an error code
* otherwise.
*/
export interface SmtpReply {
code: number;
text: string;
}
export const ACCEPTED: SmtpReply = {
code: 250,
text: '2.0.0 Message accepted',
};
export interface IngressTarget {
url: string;
/** Sent as the Host header: the endpoint is served on the api subdomain. */
host?: string | null;
secret: string;
timeoutMs: number;
}
/**
* Header fields the endpoint takes in its query string. Fallbacks match what
* the hosted receiver sends, so a message looks the same whichever one
* delivered it.
*/
export const readHeaderFields = async (message: Buffer) => {
try {
const parsed = await PostalMime.parse(message);
return {
subject: parsed.subject ?? '',
from: parsed.from?.address ?? 'anonymous@example.com',
messageId: parsed.messageId ?? 'NO-ID',
};
} catch {
// A message we cannot parse is still a message we can deliver.
return {
subject: '',
from: 'anonymous@example.com',
messageId: 'NO-ID',
};
}
};
export const buildIngressUrl = (
base: string,
params: {
to: string;
subject: string;
from: string;
messageId: string;
secret: string;
},
): URL => {
const url = new URL(base);
url.searchParams.set('subject', params.subject);
url.searchParams.set('messageid', params.messageId);
url.searchParams.set('from', params.from);
url.searchParams.set('to', params.to);
url.searchParams.set('SECRET', params.secret);
return url;
};
/**
* The endpoint without its query string. The secret travels as a query
* parameter, so the full URL is a credential and only this form may be logged.
*/
export const describeTarget = (base: string): string => {
try {
const url = new URL(base);
return `${url.origin}${url.pathname}`;
} catch {
return '<invalid ingress url>';
}
};
/**
* Anything other than "stored" and "no such user" is reported as temporary, so
* the sending server retries rather than bouncing mail over a problem at this
* end - a wrong secret included.
*/
export const replyForStatus = (status: number): SmtpReply => {
if (status >= 200 && status < 300) return ACCEPTED;
if (status === 404) return { code: 550, text: '5.1.1 No such user here' };
if (status === 413)
return { code: 552, text: '5.3.4 Message too big for system' };
return { code: 451, text: '4.3.0 Temporary server error' };
};
/** Hand one message to the ingress endpoint for one recipient. */
export const deliver = async (
message: Buffer,
recipient: string,
fields: { subject: string; from: string; messageId: string },
target: IngressTarget,
): Promise<SmtpReply> => {
const url = buildIngressUrl(target.url, {
to: recipient,
...fields,
secret: target.secret,
});
const transport = url.protocol === 'https:' ? https : http;
return await new Promise<SmtpReply>((resolve) => {
const req = transport.request(
url,
{
method: 'POST',
headers: {
'content-type': MAIL_CONTENT_TYPE,
// Required: the endpoint refuses a body of unknown length.
'content-length': String(message.byteLength),
...(target.host ? { host: target.host } : {}),
},
timeout: target.timeoutMs,
},
(res) => {
res.resume();
res.once('end', () =>
resolve(replyForStatus(res.statusCode ?? 0)),
);
},
);
// Errors from the HTTP client can carry the request URL, and that URL
// holds the secret, so none of it is surfaced.
req.once('error', () =>
resolve({ code: 451, text: '4.4.1 Ingress unavailable' }),
);
req.once('timeout', () => {
req.destroy();
resolve({ code: 451, text: '4.4.2 Ingress timed out' });
});
req.end(message);
});
};
+65
View File
@@ -0,0 +1,65 @@
import { describe, expect, test } from 'vitest';
import { acceptedRecipients } from './recipients.js';
const DOMAINS = ['example.com', 'Mail.Example.Net'];
const accept = (...addresses: string[]) =>
acceptedRecipients(addresses, DOMAINS);
describe('choosing which recipients to accept', () => {
test('accepts a configured domain regardless of case', () => {
expect(accept('dan@example.com', 'sam@MAIL.EXAMPLE.NET')).toEqual([
'dan@example.com',
'sam@MAIL.EXAMPLE.NET',
]);
});
test('refuses every other domain, which is what stops open relaying', () => {
expect(
accept(
'someone@elsewhere.test',
'sub@sub.example.com',
'spoof@example.com.evil.test',
),
).toEqual([]);
});
test('accepts nothing when no domains are configured', () => {
expect(acceptedRecipients(['dan@example.com'], [])).toEqual([]);
});
test('keeps the local part verbatim', () => {
// What a local part names is the ingress endpoint's decision, so it is
// handed over exactly as the sender wrote it.
expect(accept('Dan.Smith+tag@Example.COM')).toEqual([
'Dan.Smith+tag@Example.COM',
]);
});
test('splits on the last @, which a quoted local part may contain', () => {
expect(accept('"odd@name"@example.com')).toEqual([
'"odd@name"@example.com',
]);
});
test.each([
['no at sign', 'not-an-address'],
['empty local part', '@example.com'],
['empty domain', 'dan@'],
['whitespace', 'dan smith@example.com'],
['empty string', ''],
])('skips a malformed address (%s)', (_label, address) => {
expect(accept(address)).toEqual([]);
});
test('preserves order and drops repeats', () => {
expect(
accept('a@example.com', 'b@example.com', 'a@example.com'),
).toEqual(['a@example.com', 'b@example.com']);
});
test('treats a domain-case variant as the same recipient', () => {
expect(accept('a@example.com', 'a@EXAMPLE.COM')).toEqual([
'a@example.com',
]);
});
});
+51
View File
@@ -0,0 +1,51 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* The addresses this server will take mail for, in the order offered and
* without repeats.
*
* An address is accepted only when its domain is one of `domains` - that check
* is the whole reason this server is not an open relay. The local part is
* passed through exactly as the sender wrote it, because what it names is the
* ingress endpoint's decision, not this one's.
*/
export const acceptedRecipients = (
addresses: readonly string[],
domains: readonly string[],
): string[] => {
const accepted = domains.map((d) => d.toLowerCase());
const seen = new Set<string>();
const out: string[] = [];
for (const address of addresses) {
const at = address.lastIndexOf('@');
if (at <= 0 || at === address.length - 1) continue;
if (/\s/.test(address)) continue;
const domain = address.slice(at + 1).toLowerCase();
if (!accepted.includes(domain)) continue;
const key = `${address.slice(0, at)}@${domain}`;
if (seen.has(key)) continue;
seen.add(key);
out.push(address);
}
return out;
};
+42
View File
@@ -181,6 +181,46 @@ export interface IEmailConfig {
[key: string]: unknown;
}
/**
* Puter-addressed user mailboxes: the shared secret the ingress endpoint
* checks, and the optional local SMTP receiver that feeds it.
*
* The receiver is the self-hosted stand-in for an externally-hosted inbound
* mail path — `localServer` switches it on the same way `workers.localServer`
* switches workers to an in-process implementation. It runs as its own process,
* so nothing here affects the main server when it is off.
*/
export interface IUserEmailConfig {
/** Shared secret the ingress endpoint requires. Unset disables ingress. */
secret?: string;
/** Run the local SMTP receiver. Absent or falsy and nothing binds. */
localServer?: boolean;
/** Bind port. Default 2525 — set 25 only where the process may bind it. */
localPort?: number;
/** Bind address. Default '0.0.0.0'. */
localHost?: string;
/** Domains to accept recipients for. Required with `localServer`. */
localDomains?: string[];
/** Greeting and EHLO name. Defaults to the first accepted domain. */
localHostname?: string;
/** Ingress URL. Defaults to `<api_base_url>/email/ingress`. */
localIngressUrl?: string;
/**
* Host header to send. The ingress route is served on the `api` subdomain,
* so this must name it when `localIngressUrl` points at an internal
* address. Defaults to the host of `api_base_url`.
*/
localIngressHost?: string;
/** Envelope recipients accepted per message. Default 50. */
localMaxRecipients?: number;
/**
* Concurrent client connections. Default 20. Each message is held in memory
* while it is delivered, so this also bounds memory use.
*/
localMaxClients?: number;
[key: string]: unknown;
}
/** Prelude (https://prelude.so) Verify v2 — SMS phone verification provider. */
export interface IPreludeConfig {
/** Prelude v2 API key (sent as `Authorization: Bearer <apiKey>`). */
@@ -941,6 +981,8 @@ interface IConfigOptional {
kvCache: IKvCacheConfig;
pager: IPagerConfig;
email: IEmailConfig;
/** Puter-addressed user mailboxes and the optional local SMTP receiver. */
userEmail: IUserEmailConfig;
/** Optional — only set when SMS phone verification (Prelude) is wired in. */
prelude: IPreludeConfig;
/** Optional — only set when a ClickHouse analytics client is wired in. */
+17 -1
View File
@@ -18,6 +18,12 @@
// so proprietary extensions can live outside this repository:
//
// npm start --server=puter.com --extensions=../puter-private/gui-extensions
//
// `--smtp` builds and runs the inbound SMTP receiver instead of the backend.
// It is a separate process and exits immediately unless
// `userEmail.localServer` is set:
//
// npm start -- --smtp
import { spawn } from 'node:child_process';
import path from 'node:path';
@@ -55,9 +61,19 @@ const run = (cmd, args, opts = {}) => new Promise((resolve, reject) => {
const server = getFlag('server');
const extensions = getFlag('extensions');
const smtp =
process.argv.slice(2).includes('--smtp') ||
Boolean(process.env.npm_config_smtp);
try {
if ( server ) {
if ( smtp ) {
await run(npm, ['run', 'build:ts']);
await run(process.execPath, [
'--enable-source-maps',
'-r', './dist/src/backend/telemetry.js',
'./dist/src/backend/smtp/index.js',
]);
} else if ( server ) {
const args = ['dev-server.js', `--server=${server}`];
if ( extensions ) args.push(`--extensions=${resolveExtensionPaths(extensions)}`);
await run(process.execPath, args, {