diff --git a/src/backend/core/http/middleware/rateLimit.js b/src/backend/core/http/middleware/rateLimit.js index 34dd99983..59a96fe46 100644 --- a/src/backend/core/http/middleware/rateLimit.js +++ b/src/backend/core/http/middleware/rateLimit.js @@ -115,6 +115,31 @@ async function checkMemory(key, limit, windowMs) { return true; } +/** + * Spend up to `units` from one bucket at once and report how many fit: the + * many-unit form of `checkMemory`, for a caller that would otherwise loop. + */ +async function claimMemory(key, limit, windowMs, units) { + const now = Date.now(); + let entry = memoryWindows.get(key); + if (!entry) { + if (memoryWindows.size >= MEMORY_MAX_KEYS) { + const oldest = memoryWindows.keys().next().value; + memoryWindows.delete(oldest); + } + entry = { ts: [], windowMs }; + memoryWindows.set(key, entry); + } else { + entry.windowMs = windowMs; + } + const timestamps = entry.ts; + const cutoff = now - windowMs; + while (timestamps.length > 0 && timestamps[0] < cutoff) timestamps.shift(); + const admitted = Math.max(0, Math.min(units, limit - timestamps.length)); + for (let i = 0; i < admitted; i++) timestamps.push(now); + return admitted; +} + /** * Read a bucket's state without spending from it. For gates whose budget is * consumed by something other than the request being admitted — a failed @@ -182,6 +207,32 @@ async function checkRedis( return true; } +/** + * `checkRedis` for many units in one round trip: add them all, count, and take + * back however many landed over the limit. Same add-then-count shape, so + * concurrent claims can under-admit but never over-admit. + */ +async function claimRedis(redis, key, limit, windowMs, units) { + const redisKey = `rate:${key}`; + const now = Date.now(); + const members = Array.from( + { length: units }, + () => `${now}:${crypto.randomUUID()}`, + ); + const results = await redis + .multi() + .zremrangebyscore(redisKey, 0, now - windowMs) + .zadd(redisKey, ...members.flatMap((member) => [now, member])) + .zcard(redisKey) + .pexpire(redisKey, windowMs) + .exec(); + + const count = Number(multiResult(results, 2)); + const excess = Math.min(units, Math.max(0, count - limit)); + if (excess > 0) await redis.zrem(redisKey, ...members.slice(-excess)); + return units - excess; +} + async function peekRedis(redis, key, limit, windowMs) { const redisKey = `rate:${key}`; const results = await redis @@ -218,6 +269,31 @@ async function checkKv(kv, key, limit, windowMs) { return true; } +// `checkKv` for many units: one read, then a row per admitted unit. Newest +// first for the same reason `checkKv` reads that way. +async function claimKv(kv, key, limit, windowMs, units) { + const prefix = `rate:${key}:`; + const { res } = await kv.list({ + as: 'keys', + pattern: prefix, + limit: limit + 1, + reverse: true, + }); + const keys = Array.isArray(res) ? res : (res?.items ?? []); + const admitted = Math.max(0, Math.min(units, limit - keys.length)); + const now = Date.now(); + await Promise.all( + Array.from({ length: admitted }, () => + kv.set({ + key: `${prefix}${now}:${crypto.randomUUID()}`, + value: 1, + expireAt: Math.ceil((now + windowMs) / 1000), + }), + ), + ); + return admitted; +} + // `list` filters by TTL, so a non-expired row is in-window — same read // `checkKv` does, minus the write. async function peekKv(kv, key, limit) { @@ -395,6 +471,10 @@ function instrumentBackendPair(name, pair) { withSpan('rate_limit.peek', attrs, () => pair.peek(key, limit, windowMs), ), + claim: (key, limit, windowMs, units) => + withSpan('rate_limit.claim', attrs, () => + pair.claim(key, limit, windowMs, units), + ), acquire: (key, limit) => withSpan('rate_limit.acquire', attrs, () => pair.acquire(key, limit), @@ -405,6 +485,7 @@ function instrumentBackendPair(name, pair) { const memoryBackendPair = instrumentBackendPair('memory', { rate: checkMemory, peek: peekMemory, + claim: claimMemory, acquire: acquireMemoryConcurrent, }); @@ -453,6 +534,8 @@ export function configureRateLimit({ checkRedis(redis, key, limit, windowMs), peek: (key, limit, windowMs) => peekRedis(redis, key, limit, windowMs), + claim: (key, limit, windowMs, units) => + claimRedis(redis, key, limit, windowMs, units), acquire: (key, limit) => acquireRedisConcurrent(redis, key, limit), }); } @@ -460,6 +543,8 @@ export function configureRateLimit({ backends.kv = instrumentBackendPair('kv', { rate: (key, limit, windowMs) => checkKv(kv, key, limit, windowMs), peek: (key, limit) => peekKv(kv, key, limit), + claim: (key, limit, windowMs, units) => + claimKv(kv, key, limit, windowMs, units), acquire: (key, limit) => acquireKvConcurrent(kv, key, limit), }); } @@ -770,6 +855,25 @@ export async function checkRateLimit( ); } +/** + * `checkRateLimit` for `units` hits at once, in one backend round trip. + * Resolves to how many were admitted (0..units) and spent; the rest were not + * charged. Fails open, admitting every unit, like the rest of this module. + */ +export async function claimRateLimit(key, limit, windowMs, units, backend) { + if (!(units > 0)) return 0; + const bk = resolveBackend(backend); + try { + return await bk.claim(key, limit, windowMs, units); + } catch (err) { + console.error( + '[rate-limit] imperative claim failed, failing open:', + err, + ); + return units; + } +} + /** * Charge a route-shaped rate-limit spec from inside a handler. * diff --git a/src/backend/core/http/middleware/rateLimit.test.js b/src/backend/core/http/middleware/rateLimit.test.js index c6cb8f5b1..c71b50ad6 100644 --- a/src/backend/core/http/middleware/rateLimit.test.js +++ b/src/backend/core/http/middleware/rateLimit.test.js @@ -37,6 +37,7 @@ import { acquireDriverConcurrent, checkDriverRateLimit, checkRateLimit, + claimRateLimit, concurrencyGate, consumeRouteRateLimit, configureRateLimit, @@ -1479,6 +1480,20 @@ describe('rate + concurrent limiting — kv backend', () => { await Promise.all(held.map((slot) => slot.release())); }); + it('claims many units in one call, admitting only what fits', async () => { + const key = uniqueScope(); + expect(await claimRateLimit(key, 5, 60_000, 3)).toBe(3); + expect(await claimRateLimit(key, 5, 60_000, 3)).toBe(2); + expect(await claimRateLimit(key, 5, 60_000, 1)).toBe(0); + }); + + it('claims against a full bucket as full while expired rows are unswept', async () => { + const key = uniqueScope(); + await seedUnswept(`rate:${key}:`, 3); + expect(await claimRateLimit(key, 3, 60_000, 3)).toBe(3); + expect(await claimRateLimit(key, 3, 60_000, 1)).toBe(0); + }); + it('holds a concurrent slot until it is released', async () => { const req = { ip: '9.9.9.9', headers: {}, actor: undefined }; const opts = { limit: 1, backend: 'kv', scope: uniqueScope() }; @@ -1994,3 +2009,59 @@ describe('acquireConcurrent (imperative)', () => { configureRateLimit(); }); }); + +describe('claimRateLimit', () => { + const uniqueKey = () => `claim-${Math.random().toString(36).slice(2, 10)}`; + + describe.each([ + ['memory', () => configureRateLimit()], + [ + 'redis', + () => configureRateLimit({ default: 'redis', redis: new RedisMock() }), + ], + ])('%s backend', (_name, configure) => { + beforeEach(configure); + + it('admits every unit while under the limit', async () => { + expect(await claimRateLimit(uniqueKey(), 5, 60_000, 3)).toBe(3); + }); + + it('admits exactly up to the limit and spends only that', async () => { + const key = uniqueKey(); + expect(await claimRateLimit(key, 5, 60_000, 3)).toBe(3); + expect(await claimRateLimit(key, 5, 60_000, 3)).toBe(2); + // Nothing spent past the limit, so the bucket is exactly full. + expect(await checkRateLimit(key, 5, 60_000)).toBe(false); + expect(await checkRateLimit(key, 6, 60_000)).toBe(true); + }); + + it('admits nothing once the bucket is full', async () => { + const key = uniqueKey(); + expect(await claimRateLimit(key, 2, 60_000, 2)).toBe(2); + expect(await claimRateLimit(key, 2, 60_000, 4)).toBe(0); + }); + + it('shares its bucket with checkRateLimit', async () => { + const key = uniqueKey(); + expect(await checkRateLimit(key, 3, 60_000)).toBe(true); + expect(await claimRateLimit(key, 3, 60_000, 5)).toBe(2); + }); + }); + + it('claims nothing for zero units', async () => { + const key = uniqueKey(); + expect(await claimRateLimit(key, 1, 60_000, 0)).toBe(0); + expect(await checkRateLimit(key, 1, 60_000)).toBe(true); + }); + + it('fails open, admitting every unit, when the backend throws', async () => { + const redis = new RedisMock(); + redis.multi = () => { + throw new Error('redis down'); + }; + configureRateLimit({ default: 'redis', redis }); + const spy = vi.spyOn(console, 'error').mockImplementation(() => {}); + expect(await claimRateLimit(uniqueKey(), 1, 60_000, 4)).toBe(4); + spy.mockRestore(); + }); +}); diff --git a/src/backend/services/email/mailbox.test.ts b/src/backend/services/email/mailbox.test.ts index 7c7805064..12ce85875 100644 --- a/src/backend/services/email/mailbox.test.ts +++ b/src/backend/services/email/mailbox.test.ts @@ -70,6 +70,13 @@ describe('addresses', () => { 'dan@puterstaging.email', ]); }); + + test('a username with capitals yields lowercase addresses', () => { + expect(puterEmailAddressesOf('Dan')).toEqual([ + 'dan@puter.email', + 'dan@puterstaging.email', + ]); + }); }); describe('account eligibility', () => { diff --git a/src/backend/services/email/mailbox.ts b/src/backend/services/email/mailbox.ts index fab655196..36c3fe053 100644 --- a/src/backend/services/email/mailbox.ts +++ b/src/backend/services/email/mailbox.ts @@ -59,9 +59,13 @@ export const isPuterEmailAddress = (address: string): boolean => export const puterEmailUsername = (address: string): string => address.split('@')[0]; -/** Every address that resolves to `username`. */ +/** + * Every address that resolves to `username`, lowercased: callers compare these + * against addresses parsed out of headers, which are lowercased too, and a + * username stored with capitals would otherwise match nothing. + */ export const puterEmailAddressesOf = (username: string): string[] => - PUTER_EMAIL_DOMAINS.map((domain) => `${username}@${domain}`); + PUTER_EMAIL_DOMAINS.map((domain) => `${username.toLowerCase()}@${domain}`); /** * A temp account has neither a password nor an email - the same test diff --git a/src/docs/src/Email/sendTransactional.md b/src/docs/src/Email/sendTransactional.md index 7c347b64f..9eadb5183 100644 --- a/src/docs/src/Email/sendTransactional.md +++ b/src/docs/src/Email/sendTransactional.md @@ -42,8 +42,8 @@ A plain-text body. Shorthand for `text`; use the options form for anything else. - `attachments` (Array) - Up to **10** attachments, **25 MB** in total. Each is an object: - `filename` (String) - Required with `content`; defaults to the file's name for a `path`/`uid` attachment. - `content` (String) - The file body as base64. Mutually exclusive with `path`/`uid`. - - `path` (String) - A Puter path (`~/` allowed). The file is read on the server with the caller's file permissions, falling back to the worker owner's, and streamed rather than uploaded — prefer this over `content` for anything larger than a few hundred kilobytes. - - `uid` (String) - A Puter file uid, as an alternative to `path`. + - `path` (String) - A Puter path (`~/` allowed, meaning the caller's home). The file is read on the server with the caller's file permissions only, and streamed rather than uploaded — prefer this over `content` for anything larger than a few hundred kilobytes. + - `uid` (String) - A Puter file uid, as an alternative to `path`. Read with the caller's file permissions, falling back to the worker owner's — use a uid to attach a file from the worker owner's storage. - `contentType` (String) - MIME type. Detected from the file for `path`/`uid` attachments. - `cid` (String) - Content-ID that makes the attachment an inline part of the `html` body: with `cid: 'logo'`, `` shows it. Printable ASCII with no whitespace or angle brackets. Inline parts are billed like any other attachment. @@ -52,7 +52,7 @@ A plain-text body. Shorthand for `text`; use the options form for anything else. A `Promise` that resolves to an object: - `messageId` (String | null) - The `Message-ID` of the first delivery: the transport's for a relayed copy, the stored message's for a Puter mailbox. -- `cost` (Number) - What the send was charged, in microcents. +- `cost` (Number) - What the send was charged, in microcents: one delivery's price for each recipient in neither `failed` nor `suppressed`. - `suppressed` (Array) - Recipients (lowercased) dropped because they unsubscribed from your app's mail. The message went to the others. - `failed` (Array) - Recipients (lowercased) whose delivery failed. Everyone else got their copy — retry with just these addresses. @@ -65,7 +65,7 @@ In case of an error, the `Promise` rejects with `{ message, code }`. Codes you m - `subscription_required` (402) - The calling account is not on a paid plan. - `insufficient_funds` (402) - The calling account has no usage credit left for this send. - `bad_request` (400) - Invalid arguments: a bad address, a missing body, too many recipients, an invalid attachment. -- `not_found` (404) - Every recipient was a Puter address with no mailbox to receive in. +- `not_found` (404) - An attachment's `path`/`uid` could not be read; or every recipient was a Puter address with no mailbox to receive in. - `too_many_requests` (429) - Over the rate limit; see [Rate Limits and Quotas](/rate-limits-and-quotas/#email). ## Recipients on Puter @@ -86,7 +86,7 @@ Every message carries an unsubscribe link and a report-abuse link. Opting out is ## Limits and cost -Sends are rate-limited per calling account; the numbers are on [Rate Limits and Quotas](/rate-limits-and-quotas/#email). A send costs a flat charge per message plus a per-byte charge for attachments, and is charged to the calling account. A send that reaches nobody is not charged. +Sends are rate-limited per calling account; the numbers are on [Rate Limits and Quotas](/rate-limits-and-quotas/#email). Each recipient's delivery costs a flat charge plus a per-byte charge for the attachments it carries, so a send to three recipients costs three deliveries. Only deliveries that succeed are charged — retrying the `failed` addresses never pays twice — and a send that reaches nobody is not charged. The calling account pays. ## Examples @@ -125,6 +125,10 @@ router.post('/api/receipt', async ({ request, user }) => { Attach a file and show an inline image ```js +// The uid of a file in the worker owner's storage, e.g. from +// `(await me.puter.fs.stat('~/Public/logo.png')).uid`. +const LOGO_UID = '...'; + router.post('/api/invoice', async ({ request, user }) => { const { to } = await request.json(); return await user.puter.email.sendTransactional({ @@ -132,9 +136,9 @@ router.post('/api/invoice', async ({ request, user }) => { subject: 'Your invoice', html: 'Acme

Your invoice is attached.

', attachments: [ - // Streamed from the worker owner's storage; shown inline via the cid. - { path: '~/Public/logo.png', cid: 'logo' }, - // Streamed from the user's own storage as a regular attachment. + // A uid falls back to the worker owner's storage; shown inline via the cid. + { uid: LOGO_UID, cid: 'logo' }, + // A path is always the caller's: streamed from the user's own storage. { path: '~/Documents/invoice.pdf' }, ], emailAccessToken: me.puter.authToken, diff --git a/src/puter-js/src/modules/Email/ComposerLib.js b/src/puter-js/src/modules/Email/ComposerLib.js index e6a83dbc9..a093a7726 100644 --- a/src/puter-js/src/modules/Email/ComposerLib.js +++ b/src/puter-js/src/modules/Email/ComposerLib.js @@ -2,18 +2,23 @@ // Multipart email files are handled as String'd FormData to save on Multipart parsing logic. The primary goal of this library // is not necessarily high performance but high maintanability. +import { PuterJSError } from '../../lib/PuterJSError.js'; + +/** Every refusal here is the caller's input; the SDK error contract wants a code. */ +const invalid = (message) => new PuterJSError(message, 'invalid_request'); + /** * One attachment: either inline base64 `content`, or a Puter FS reference - * (`path`/`uid`) read server-side with the caller's — falling back to the - * authorizing worker's — file permissions. FS references are streamed from + * (`path`/`uid`) read server-side with the caller's file permissions; a `uid` + * falls back to the authorizing worker's. FS references are streamed from * storage and never travel through the request, so prefer them for anything * larger than a few hundred kilobytes. * * @typedef {Object} EmailAttachment * @property {string} [filename] Required with `content`; defaults to the file's name for FS refs. * @property {string} [content] Base64 file body. Mutually exclusive with `path`/`uid`. - * @property {string} [path] Puter FS path (supports `~/`). Mutually exclusive with `content`. - * @property {string} [uid] Puter FS entry uid. Mutually exclusive with `content`. + * @property {string} [path] Puter FS path (supports `~/`, the caller's home). Mutually exclusive with `content`. + * @property {string} [uid] Puter FS entry uid; the only way to attach a file of the worker owner's. Mutually exclusive with `content`. * @property {string|undefined} [cid] Email content ID for inline images * @property {string} [contentType] MIME type of the attachment. */ @@ -40,25 +45,103 @@ export function getRFC822DateUTC(date = new Date()) { export function emlHeader(key, value) { if (!isASCII(key)) { - throw new Error("eml header key " + key + " is not valid ASCII"); + throw invalid("eml header key " + key + " is not valid ASCII"); } if (Array.isArray(value)) { value.forEach((val) => { if (!isASCII(val)) { - throw new Error("eml header value " + val + " is not valid ASCII"); + throw invalid("eml header value " + val + " is not valid ASCII"); } }) return `${key}: ${value.join(', ')}\r\n`; } else { if (!isASCII(value)) { - throw new Error("eml header value " + value + " is not valid ASCII"); + throw invalid("eml header value " + value + " is not valid ASCII"); } return `${key}: ${value}\r\n`; } } +// UTF-8 bytes per encoded word: 45 bytes is 60 base64 characters, which with +// the `=?UTF-8?B?` and `?=` wrapping stays inside RFC 2047's 75. +const ENCODED_WORD_BYTES = 45; + +const bytesToBase64 = (bytes) => { + let binary = ''; + for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]); + return btoa(binary); +}; + +/** + * Header text as RFC 2047 encoded words: printable ASCII comes back as is, + * anything else as `=?UTF-8?B?...?=` words split on character boundaries and + * folded onto continuation lines. Control characters are refused either way: + * they have no business in a header, encoded or not. + * + * @param {string} text + * @returns {string} + */ +export function encodeWord(text) { + if (typeof text !== 'string' || /[\x00-\x1f\x7f]/.test(text)) { + throw invalid("eml header value " + text + " is not valid"); + } + if (isASCII(text)) return text; + + const encoder = new TextEncoder(); + const words = []; + let pending = []; + for (const char of text) { + const bytes = encoder.encode(char); + if (pending.length + bytes.length > ENCODED_WORD_BYTES) { + words.push(pending); + pending = []; + } + pending.push(...bytes); + } + if (pending.length > 0) words.push(pending); + return words.map(word => `=?UTF-8?B?${bytesToBase64(word)}?=`).join('\r\n '); +} + +/** + * One address-list entry, its display name encoded when it needs to be. The + * address itself has to stay ASCII: mail without SMTPUTF8 has no encoding + * for it. + * + * @param {string} entry `addr@host` or `Name ` + * @returns {string} + */ +function encodeAddress(entry) { + if (isASCII(entry)) return entry; + const match = /^\s*(.*?)\s*<([^<>]*)>\s*$/.exec(String(entry)); + if (!match || !isASCII(match[2])) { + throw invalid("eml address " + entry + " is not valid ASCII"); + } + const name = match[1].replace(/^"(.*)"$/, '$1').replace(/\\(.)/g, '$1'); + return `${encodeWord(name)} <${match[2]}>`; +} + +/** A free-text header (Subject), RFC 2047 encoded when not plain ASCII. */ +export function textHeader(key, value) { + if (!isASCII(key)) { + throw invalid("eml header key " + key + " is not valid ASCII"); + } + return `${key}: ${encodeWord(value)}\r\n`; +} + +/** + * An address-list header (To, Cc, Bcc, Reply-To, From) with non-ASCII display + * names encoded. A string is one entry, or an ASCII list passed through as is. + */ +export function addressHeader(key, value) { + if (!isASCII(key)) { + throw invalid("eml header key " + key + " is not valid ASCII"); + } + const entries = Array.isArray(value) ? value : [value]; + return `${key}: ${entries.map(encodeAddress).join(', ')}\r\n`; +} + export function determineTopLevelMimeType({ attachments = [], text, html }) { const inlineAttachments = attachments.filter(a => a.cid); const regularAttachments = attachments.filter(a => !a.cid); @@ -78,7 +161,7 @@ export function determineTopLevelMimeType({ attachments = [], text, html }) { if (html) { return { isMultiPart: false, mimeType: 'text/html; charset=UTF-8' }; } - throw new Error('Email must have text, html, or an attachment'); + throw invalid('Email must have text, html, or an attachment'); } @@ -100,21 +183,22 @@ export const isASCII = (str) => /^[\x20-\x7E]*$/.test(str); /** * Composes a multipart email attachment - * - * @param {EmailAttachment} attachment + * + * @param {EmailAttachment} attachment + * @param {import('../../index.js').Puter} [puter] Reads `path` attachments; the module's own instance. */ -export async function composeAttachment({ cid, content, contentType, filename, path, uid }) { +export async function composeAttachment({ cid, content, contentType, filename, path, uid }, puter = globalThis.puter) { if (path && content) { - throw new Error("Path and Content are mutually exclusive"); + throw invalid("Path and Content are mutually exclusive"); } if (!path && !content && !uid) { - throw new Error("Attachment requires content, path, or uid"); + throw invalid("Attachment requires content, path, or uid"); } if (uid) { - throw new Error("uid attachments are not supported yet"); + throw invalid("uid attachments are not supported yet"); } if (contentType && !filename) { - throw new Error("filename parameter required for contentType"); + throw invalid("filename parameter required for contentType"); } let b64emailContent = content; @@ -129,12 +213,12 @@ export async function composeAttachment({ cid, content, contentType, filename, p } if (!filename) { - throw new Error("Attachment's filename cannot be determined automatically. Please provide filename!"); + throw invalid("Attachment's filename cannot be determined automatically. Please provide filename!"); } filename = filename.replaceAll('\\', '\\\\'); filename = filename.replaceAll('"', '\\"'); if (!isASCII(filename)) { - throw new Error("filename must be ascii"); + throw invalid("filename must be ascii"); } if (contentType) { @@ -142,7 +226,7 @@ export async function composeAttachment({ cid, content, contentType, filename, p } if (!mimeType) { - throw new Error("Attachment's mimetype cannot be determined automatically. Please provide contentType!"); + throw invalid("Attachment's mimetype cannot be determined automatically. Please provide contentType!"); } mimeType += ';name="' + filename + '"'; @@ -168,34 +252,36 @@ export function combineParts(parts, boundary) { } /** - * - * @param {EmailComposeOptions} param0 + * + * @param {EmailComposeOptions} param0 + * @param {import('../../index.js').Puter} [puter] The module's own instance: who the mail is from, and + * whose storage `path` attachments are read from. */ -export async function compose({ from, to, cc, bcc, subject, replyTo, attachments = [], text, html }) { +export async function compose({ from, to, cc, bcc, subject, replyTo, attachments = [], text, html }, puter = globalThis.puter) { // --- Header constructing logic --- let headerLines = "MIME-Version: 1.0\r\n"; headerLines += emlHeader('Date', getRFC822DateUTC()); if (subject) { - headerLines += emlHeader('Subject', subject); + headerLines += textHeader('Subject', subject); } if (to) { - headerLines += emlHeader('To', to); + headerLines += addressHeader('To', to); } else { - throw new Error("Cannot compose email without recipient!"); + throw invalid("Cannot compose email without recipient!"); } if (cc) { - headerLines += emlHeader('cc', cc); + headerLines += addressHeader('cc', cc); } if (bcc) { - headerLines += emlHeader('bcc', bcc); + headerLines += addressHeader('bcc', bcc); } if (replyTo) { - headerLines += emlHeader('Reply-To', replyTo); + headerLines += addressHeader('Reply-To', replyTo); } if (from) { - headerLines += emlHeader('From', from); + headerLines += addressHeader('From', from); } else { // Add From Block (hardcoded to puter.email for now) const userinfo = await puter.getUser(); @@ -229,8 +315,8 @@ export async function compose({ from, to, cc, bcc, subject, replyTo, attachments const inlineAttachments = attachments.filter(a => a.cid); const regularAttachments = attachments.filter(a => !a.cid); - const inlineParts = await Promise.all(inlineAttachments.map(composeAttachment)); - const regularParts = await Promise.all(regularAttachments.map(composeAttachment)); + const inlineParts = await Promise.all(inlineAttachments.map(a => composeAttachment(a, puter))); + const regularParts = await Promise.all(regularAttachments.map(a => composeAttachment(a, puter))); // Build the text/html content as one part. Only wrap it in its own nested // multipart/alternative (fresh boundary) when both text and html are present. diff --git a/src/puter-js/src/modules/Email/ComposerLib.test.js b/src/puter-js/src/modules/Email/ComposerLib.test.js index db73e119a..551ca1f5b 100644 --- a/src/puter-js/src/modules/Email/ComposerLib.test.js +++ b/src/puter-js/src/modules/Email/ComposerLib.test.js @@ -502,8 +502,63 @@ describe('compose', () => { await expect(compose({ ...baseOptions, attachments: [] })).rejects.toThrow(/text, html, or an attachment/); }); - it('rejects a non-ASCII subject', async () => { - await expect(compose({ ...baseOptions, subject: 'héllo', text: 'hi' })).rejects.toThrow(/not valid ASCII/); + it('encodes a non-ASCII subject as RFC 2047 words instead of refusing it', async () => { + const eml = await compose({ ...baseOptions, subject: 'héllo', text: 'hi' }); + expect(eml).toContain('Subject: =?UTF-8?B?aMOpbGxv?=\r\n'); + expect(isASCII(eml.split('\r\n\r\n')[0].replace(/\r\n/g, ''))).toBe(true); + }); + + it('folds a long non-ASCII subject into words of at most 75 characters', async () => { + const subject = 'Réunion — 東京 🎉 '.repeat(12); + const eml = await compose({ ...baseOptions, subject, text: 'hi' }); + const head = eml.split('\r\n\r\n')[0]; + const subjectLines = head.slice(head.indexOf('Subject: ')).split('\r\n'); + const words = [subjectLines[0].slice('Subject: '.length)]; + for ( const line of subjectLines.slice(1) ) { + if ( ! line.startsWith(' ') ) break; + words.push(line.slice(1)); + } + expect(words.length).toBeGreaterThan(1); + for ( const word of words ) { + expect(word).toMatch(/^=\?UTF-8\?B\?[A-Za-z0-9+/=]+\?=$/); + expect(word.length).toBeLessThanOrEqual(75); + } + }); + + it('refuses control characters in a subject, encoded or not', async () => { + await expect(compose({ ...baseOptions, subject: 'hi\r\nBcc: x@y.z', text: 'hi' })) + .rejects.toMatchObject({ code: 'invalid_request' }); + await expect(compose({ ...baseOptions, subject: 'héllo\r\nBcc: x@y.z', text: 'hi' })) + .rejects.toMatchObject({ code: 'invalid_request' }); + }); + + it('refuses a non-ASCII address while allowing a non-ASCII display name', async () => { + await expect(compose({ ...baseOptions, to: 'ü@example.com', text: 'hi' })) + .rejects.toMatchObject({ code: 'invalid_request' }); + await expect(compose({ ...baseOptions, to: 'Jürgen <ü@example.com>', text: 'hi' })) + .rejects.toMatchObject({ code: 'invalid_request' }); + await expect(compose({ ...baseOptions, to: 'Jürgen ', text: 'hi' })).resolves.toBeTypeOf('string'); + }); + + it('rejects with a coded PuterJSError, per the SDK error contract', async () => { + await expect(compose({ subject: 'hi', text: 'hi' })) + .rejects.toMatchObject({ name: 'PuterJSError', code: 'invalid_request' }); + await expect(composeAttachment({ uid: 'uid-1' })) + .rejects.toMatchObject({ code: 'invalid_request' }); + expect(() => emlHeader('Subject', 'héllo')).toThrow(expect.objectContaining({ code: 'invalid_request' })); + }); + + it('uses the instance it is given rather than the global one', async () => { + const own = { + fs: { read: vi.fn(async () => new Blob([PNG_BYTES], { type: 'image/png' })) }, + getUser: vi.fn(async () => ({ username: 'carol' })), + }; + const eml = await compose({ to: 'bob@example.com', subject: 'hi', html: '

', attachments: [{ path: '~/x.png', cid: 'c' }] }, own); + expect(eml).toContain('From: carol@puter.email\r\n'); + expect(own.fs.read).toHaveBeenCalledWith('~/x.png'); + expect(own.getUser).toHaveBeenCalled(); + expect(fsRead).not.toHaveBeenCalled(); + expect(getUser).not.toHaveBeenCalled(); }); }); @@ -513,6 +568,24 @@ describe('compose round-trips through a MIME parser', () => { // The parser terminates every text body with a newline of its own. const bodyOf = (str) => str?.replace(/\n$/, ''); + it('recovers a UTF-8 subject and display names', async () => { + const subject = 'Grüße ✓ — 東京 🎉 '.repeat(6).trim(); + const parsed = await parse({ + subject, + to: ['"José Pérez" ', 'plain@example.com'], + cc: 'Zoë ', + from: 'Ålice ', + text: 'hi', + }); + expect(parsed.subject).toBe(subject); + expect(parsed.to).toEqual([ + { address: 'jose@example.com', name: 'José Pérez' }, + { address: 'plain@example.com', name: '' }, + ]); + expect(parsed.cc).toEqual([{ address: 'zoe@example.com', name: 'Zoë' }]); + expect(parsed.from).toEqual({ address: 'alice@puter.email', name: 'Ålice' }); + }); + it('recovers a plain text body', async () => { const parsed = await parse({ text: 'hello there' }); expect(bodyOf(parsed.text)).toBe('hello there'); diff --git a/src/puter-js/src/modules/Email/Email.js b/src/puter-js/src/modules/Email/Email.js index b22f31611..e278742f4 100644 --- a/src/puter-js/src/modules/Email/Email.js +++ b/src/puter-js/src/modules/Email/Email.js @@ -32,7 +32,8 @@ import { list } from './list.js'; * * @typedef {Object} EmailSendResult * @property {string | null} messageId First transport message id reported for this send, when available. - * @property {number} cost Total charge for this send, in microcents. + * @property {number} cost Total charge for this send, in microcents: one delivery's price for each + * recipient that got a copy. * @property {string[]} suppressed Recipients omitted because they opted out of this app's mail. * @property {string[]} failed Recipients whose delivery attempt failed. Everyone else got their copy — * retry with just these addresses. A send where every delivery fails rejects instead. @@ -82,7 +83,8 @@ const preprocessSendArgs = (args) => { * // Inline or Puter-FS attachments: * attachments: [ * { filename, content, contentType }, // content = base64 - * { path: '~/Documents/report.pdf' }, // streamed server-side + * { path: '~/Documents/report.pdf' }, // the caller's file, streamed server-side + * { uid: LOGO_UID }, // a uid may also name a file of the worker owner's * ], * }); * }); @@ -141,6 +143,9 @@ export class EmailModule extends PuterModule { iface: 'puter-transactional-email', method: 'sendTransactional', argNames: ['to', 'subject', 'body'], + // This module's instance, not `globalThis.puter`: in a worker the two + // differ (`user.puter` vs `me.puter`), and the caller is the one billed. + puter: this.puter, preprocess: preprocessSendArgs, upgradePrompt: { method: 'puter.email.sendTransactional', @@ -157,7 +162,15 @@ export class EmailModule extends PuterModule { * @returns {Promise} */ send = async (options) => { - const req = await fetchUrl(`${this.APIOrigin}/email/send`, { method: "POST", includePuterAuth: true, body: new Blob([await compose(options)], { type: 'message/rfc822' }) }); + // `includePuterAuth` reads the global instance's token, so any other + // instance (a worker's `user.puter`) sends its own. + const isGlobal = this.puter === globalThis.puter; + const req = await fetchUrl(`${this.APIOrigin}/email/send`, { + method: "POST", + includePuterAuth: isGlobal, + ...(isGlobal ? {} : { authToken: this.authToken }), + body: new Blob([await compose(options, this.puter)], { type: 'message/rfc822' }), + }); const result = await req.json(); if ( ! req.ok ) { promptIfUpgradeRequired( diff --git a/src/puter-js/src/modules/Email/Email.test.js b/src/puter-js/src/modules/Email/Email.test.js new file mode 100644 index 000000000..8629af5a3 --- /dev/null +++ b/src/puter-js/src/modules/Email/Email.test.js @@ -0,0 +1,77 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +/** + * The email methods have to act for the instance they hang off, not for + * `globalThis.puter`: in a worker, `user.puter` and `me.puter` are different + * identities, and the one making the call is the one billed and rate-limited. + */ + +const captured = vi.hoisted(() => ({ specs: [], fetches: [] })); + +vi.mock('../../lib/utils.js', async (importOriginal) => ({ + ...(await importOriginal()), + makeDriverMethod: (spec) => { + captured.specs.push(spec); + return async () => ({}); + }, +})); + +vi.mock('../../lib/networkUtils.js', async (importOriginal) => ({ + ...(await importOriginal()), + fetchUrl: async (url, opts) => { + captured.fetches.push({ url, opts }); + return { ok: true, status: 200, json: async () => ({ delivered: [] }) }; + }, +})); + +const { EmailModule } = await import('./Email.js'); + +const origPuter = globalThis.puter; + +const makePuter = (username, authToken) => ({ + APIOrigin: 'https://api.example.test', + authToken, + getUser: vi.fn(async () => ({ username })), + fs: { read: vi.fn() }, +}); + +beforeEach(() => { + captured.specs.length = 0; + captured.fetches.length = 0; + globalThis.puter = makePuter('global', 'global-token'); +}); + +afterEach(() => { + globalThis.puter = origPuter; +}); + +describe('EmailModule binds to its own instance', () => { + it('hands sendTransactional its own puter, not the global one', () => { + const own = makePuter('caller', 'caller-token'); + new EmailModule(own); + const spec = captured.specs.find(s => s.method === 'sendTransactional'); + expect(spec.puter).toBe(own); + }); + + it('sends from, and authenticates as, a non-global instance', async () => { + const own = makePuter('caller', 'caller-token'); + const email = new EmailModule(own); + await email.send({ to: 'bob@example.com', subject: 'hi', text: 'hi' }); + + expect(own.getUser).toHaveBeenCalled(); + expect(globalThis.puter.getUser).not.toHaveBeenCalled(); + const [{ url, opts }] = captured.fetches; + expect(url).toBe('https://api.example.test/email/send'); + expect(opts.includePuterAuth).toBe(false); + expect(opts.authToken).toBe('caller-token'); + expect(await opts.body.text()).toContain('From: caller@puter.email\r\n'); + }); + + it('keeps the live-token path for the global instance', async () => { + const email = new EmailModule(globalThis.puter); + await email.send({ to: 'bob@example.com', subject: 'hi', text: 'hi' }); + const [{ opts }] = captured.fetches; + expect(opts.includePuterAuth).toBe(true); + expect(opts.authToken).toBeUndefined(); + }); +}); diff --git a/src/puter-js/src/modules/Email/lib/mailbox.js b/src/puter-js/src/modules/Email/lib/mailbox.js index 3f3763b1f..a42243fa0 100644 --- a/src/puter-js/src/modules/Email/lib/mailbox.js +++ b/src/puter-js/src/modules/Email/lib/mailbox.js @@ -122,15 +122,24 @@ export const toSummary = (entry, folder) => { }; /** - * Where a listing resumes: the day folder being walked and, while it still has - * pages, the readdir cursor within it. + * Where a listing resumes: the day folder being walked; while it still has + * pages, the readdir cursor within it; and the readdir cursor of the page of + * day folders that day was read from, so resuming does not re-list every day + * newer than it. A cursor without that last part (issued before it existed) + * walks the days from the top, which is only slower. * - * @typedef {{ folder: EmailFolder, day: string, fsCursor?: string }} ListPosition + * @typedef {{ folder: EmailFolder, day: string, fsCursor?: string, daysCursor?: string }} ListPosition */ /** @param {ListPosition} position @returns {string} */ -export const encodeCursor = ({ folder, day, fsCursor }) => { - const payload = { v: 1, f: folder, d: day, ...(fsCursor ? { c: fsCursor } : {}) }; +export const encodeCursor = ({ folder, day, fsCursor, daysCursor }) => { + const payload = { + v: 1, + f: folder, + d: day, + ...(fsCursor ? { c: fsCursor } : {}), + ...(daysCursor ? { dc: daysCursor } : {}), + }; return btoa(JSON.stringify(payload)); }; @@ -153,7 +162,12 @@ export const decodeCursor = (cursor, folder) => { if ( payload.f !== folder ) { throw new PuterJSError('Cursor does not match the requested folder', 'invalid_request'); } - return { folder, day: payload.d, ...(typeof payload.c === 'string' ? { fsCursor: payload.c } : {}) }; + return { + folder, + day: payload.d, + ...(typeof payload.c === 'string' ? { fsCursor: payload.c } : {}), + ...(typeof payload.dc === 'string' ? { daysCursor: payload.dc } : {}), + }; }; /** A readdir on a folder that was never created. @param {unknown} e */ diff --git a/src/puter-js/src/modules/Email/list.js b/src/puter-js/src/modules/Email/list.js index aa83c2c79..5fdcf7a01 100644 --- a/src/puter-js/src/modules/Email/list.js +++ b/src/puter-js/src/modules/Email/list.js @@ -12,36 +12,49 @@ import { /** @typedef {import('../../lib/types.js').ListPage} EmailListPage */ /** @typedef {import('./lib/mailbox.js').ListPosition} ListPosition */ -// Page size for the SDK's own walks over the mailbox's day folders. -const DIRECTORY_PAGE_LIMIT = 1000; +// Day folders read per directory page. Small on purpose: a listing resumes +// from the page its cursor's day came from, so this bounds what every later +// page re-reads before reaching its own day. About a month of a mailbox that +// gets mail daily. +const DAY_PAGE_LIMIT = 30; /** - * Every day folder in `folder`, newest first. A mailbox that was never - * written to has no folder at all, which lists as empty. + * The day folders in `folder`, newest first, read a directory page at a time + * starting at `daysCursor`. Each day comes with the cursor of the page it was + * read from, which is where a later listing can pick the walk back up. A + * mailbox that was never written to has no folder at all, which walks as + * empty. * - * @this {import('./Email.js').EmailModule} + * @param {import('../../index.js').Puter} puter * @param {EmailFolder} folder - * @returns {Promise} + * @param {string | undefined} daysCursor + * @returns {AsyncGenerator<{ day: string, daysCursor?: string }>} */ -async function listDays (folder) { - const fetchPage = pageParams => this.puter.fs.readdir({ - path: folderRoot(folder), - limit: DIRECTORY_PAGE_LIMIT, - sortBy: 'name', - sortOrder: 'desc', - ...pageParams, - }); - const days = []; - try { - for await ( const page of iteratePages(fetchPage) ) { - for ( const entry of page.items ) { - if ( entry.is_dir && isDayName(entry.name) ) days.push(entry.name); +async function* walkDays (puter, folder, daysCursor) { + let cursor = daysCursor ?? null; + while ( true ) { + /** @type {{ items: Array>, cursor?: string }} */ + let page; + try { + page = await puter.fs.readdir({ + path: folderRoot(folder), + limit: DAY_PAGE_LIMIT, + sortBy: 'name', + sortOrder: 'desc', + cursor, + }); + } catch (e) { + if ( ! isMissingDirectory(e) ) throw e; + return; + } + for ( const entry of page.items ) { + if ( entry.is_dir && isDayName(entry.name) ) { + yield { day: entry.name, ...(cursor ? { daysCursor: cursor } : {}) }; } } - } catch (e) { - if ( ! isMissingDirectory(e) ) throw e; + if ( ! page.cursor ) return; + cursor = page.cursor; } - return days.sort().reverse(); } /** @@ -56,18 +69,20 @@ async function listDays (folder) { * @returns {Promise} */ async function fetchPage (folder, limit, position) { - const days = await listDays.call(this, folder); + const days = walkDays(this.puter, folder, position?.daysCursor); /** @type {EmailSummary[]} */ const items = []; // Resume at the cursor's day; a day folder deleted since the cursor was // issued means continuing from the next older one. - let dayIndex = position ? days.findIndex(day => day <= position.day) : 0; - if ( dayIndex === -1 ) return { items }; - let fsCursor = position && days[dayIndex] === position.day ? position.fsCursor : undefined; + let next = await days.next(); + while ( position && ! next.done && next.value.day > position.day ) { + next = await days.next(); + } + let fsCursor = position && ! next.done && next.value.day === position.day ? position.fsCursor : undefined; - while ( dayIndex < days.length && items.length < limit ) { - const day = days[dayIndex]; + while ( ! next.done && items.length < limit ) { + const { day, daysCursor } = next.value; // Keep reading this day until the page is full or the day is spent: // entries that are not mail objects are skipped, and skipping must // not shorten the page while the same day still has messages. @@ -94,13 +109,14 @@ async function fetchPage (folder, limit, position) { if ( ! fsCursor ) break; } if ( fsCursor ) { - return { items, cursor: encodeCursor({ folder, day, fsCursor }) }; + return { items, cursor: encodeCursor({ folder, day, fsCursor, daysCursor }) }; } - dayIndex += 1; + next = await days.next(); } - if ( dayIndex < days.length ) { - return { items, cursor: encodeCursor({ folder, day: days[dayIndex] }) }; + if ( ! next.done ) { + const { day, daysCursor } = next.value; + return { items, cursor: encodeCursor({ folder, day, daysCursor }) }; } return { items }; } diff --git a/src/puter-js/src/modules/Email/mailbox.test.js b/src/puter-js/src/modules/Email/mailbox.test.js index 531a37015..ecea17207 100644 --- a/src/puter-js/src/modules/Email/mailbox.test.js +++ b/src/puter-js/src/modules/Email/mailbox.test.js @@ -79,6 +79,13 @@ describe('cursors', () => { expect(decodeCursor(encodeCursor({ folder: 'sent', day: '2024-03-15' }), 'sent')).toEqual({ folder: 'sent', day: '2024-03-15' }); }); + it('carries the day-listing cursor, and reads one issued without it', () => { + const position = { folder: 'inbox', day: '2024-03-15', fsCursor: 'abc', daysCursor: 'dc1' }; + expect(decodeCursor(encodeCursor(position), 'inbox')).toEqual(position); + const legacy = btoa(JSON.stringify({ v: 1, f: 'inbox', d: '2024-03-15', c: 'abc' })); + expect(decodeCursor(legacy, 'inbox')).toEqual({ folder: 'inbox', day: '2024-03-15', fsCursor: 'abc' }); + }); + it('treats null and undefined as the first page', () => { expect(decodeCursor(null, 'inbox')).toBeUndefined(); expect(decodeCursor(undefined, 'inbox')).toBeUndefined(); @@ -208,6 +215,61 @@ describe('list()', () => { expect(calls.every(c => c.path.startsWith('~/.mail/sent'))).toBe(true); }); + it('resumes the day walk where the cursor left it instead of re-listing from the top', async () => { + // 100 days, one message each: far more day folders than one page of them. + const tree = { '~/.mail/objects': [] }; + const dayIds = []; + for ( let i = 0; i < 100; i++ ) { + const ms = Date.UTC(2024, 0, 1) + i * 24 * 60 * 60 * 1000; + const day = new Date(ms).toISOString().slice(0, 10); + const id = uuidAt(ms + 1000); + tree['~/.mail/objects'].push(day); + tree[`~/.mail/objects/${ day }`] = [`${ id }--${ toB64Url(day) }`]; + dayIds.unshift(id); + } + const { fs, calls } = fakeFs(tree); + const email = moduleOver(fs); + + const seen = []; + let cursor = null; + do { + const before = calls.length; + const page = await email.list.call(email, { limit: 7, cursor }); + seen.push(...page.items.map(m => m.id)); + const dayListings = calls.slice(before).filter(c => c.path === '~/.mail/objects'); + // Never more than the page holding the cursor's day plus the one + // after it, however deep into the mailbox the cursor points. + expect(dayListings.length).toBeLessThanOrEqual(2); + if ( cursor ) { + const resumedFrom = decodeCursor(cursor, 'inbox').daysCursor ?? null; + expect(dayListings[0].cursor).toBe(resumedFrom); + } + cursor = page.cursor; + } while ( cursor ); + expect(seen).toEqual(dayIds); + // Only pages whose day sits in the first page of day folders start + // there; the deeper ones resumed mid-folder. + const fromTop = calls.filter(c => c.path === '~/.mail/objects' && ! c.cursor); + expect(fromTop.length).toBeLessThanOrEqual(Math.ceil(30 / 7) + 1); + expect(calls.some(c => c.path === '~/.mail/objects' && c.cursor)).toBe(true); + }); + + it('resumes from a cursor issued before day-listing cursors existed', async () => { + const { fs } = fakeFs(inboxTree); + const email = moduleOver(fs); + const legacy = btoa(JSON.stringify({ v: 1, f: 'inbox', d: '2024-03-15' })); + const page = await email.list.call(email, { limit: 10, cursor: legacy }); + expect(page.items.map(m => m.id)).toEqual([ids.d15a, ids.d14b, ids.d14a]); + }); + + it('continues from the next older day when the cursor day was deleted', async () => { + const { fs } = fakeFs({ ...inboxTree, '~/.mail/objects': ['2024-03-14', '2024-03-16'] }); + const email = moduleOver(fs); + const gone = encodeCursor({ folder: 'inbox', day: '2024-03-15', fsCursor: '5' }); + const page = await email.list.call(email, { limit: 10, cursor: gone }); + expect(page.items.map(m => m.id)).toEqual([ids.d14b, ids.d14a]); + }); + it('rejects offset, includeTotal, and a foreign cursor before any request', async () => { const { fs, calls } = fakeFs(inboxTree); const email = moduleOver(fs);