fix: keep an undelivered broadcast event instead of dropping it

The outbound queue was cleared before the send, so a peer that timed out
took its events with it. Most were survivable — a lost cache invalidation
heals when the entry's TTL lapses. A revoke's flat-perm invalidation is
not: grant-path entries carry no expiry, so a peer went on serving a
withdrawn grant until something else wrote that key.

Failed sends now go back on the queue, which the existing flush timer
retries. Anything queued since wins over the retry, and the queue is
bounded at 10,000 with the oldest dropped first, so a peer that stays
down cannot grow it without limit. Each retry is signed at send time, so
it is not rejected against the replay window.
This commit is contained in:
Juan Castro
2026-08-18 15:31:59 -04:00
parent 038b9ea550
commit 75dbc58628
2 changed files with 53 additions and 0 deletions
@@ -603,6 +603,33 @@ describe('BroadcastService outbound flush', () => {
);
expect(occurrences).toBe(1);
});
// A dropped invalidation can outlive every TTL meant to heal it.
it('sends a failed event again instead of dropping it', async () => {
const cacheKey = 'requeue-test';
const carries = (call: unknown) => {
const body = (call as [{ data?: string }])[0]?.data;
return typeof body === 'string' && body.includes(cacheKey);
};
axiosRequestMock.mockRejectedValue(new Error('peer unreachable'));
server.clients.event.emit(
'outer.cacheUpdate' as never,
{ cacheKey: [cacheKey] } as never,
{},
);
await waitForFlush(() => axiosRequestMock.mock.calls.some(carries));
axiosRequestMock.mockReset();
axiosRequestMock.mockResolvedValue({
status: 200,
statusText: 'OK',
data: 'ok',
});
// Nothing new is emitted — only the retry can satisfy this.
await waitForFlush(() => axiosRequestMock.mock.calls.some(carries));
});
});
// -- Header validation ------------------------------------------------
@@ -98,6 +98,8 @@ export class BroadcastService extends PuterService {
#webhookReplayWindowSeconds = 300;
#outboundFlushMs = 2000;
/** Bound on re-queued events, so a peer that stays down can't grow it. */
#outboundMaxQueued = 10_000;
#webhookProtocol: 'http' | 'https' = 'https';
#webhookHostHeader: string | null = null;
/** Self-signed certs are common between Puter nodes — accept them. */
@@ -350,17 +352,21 @@ export class BroadcastService extends PuterService {
const events = [...this.#outboundEventsByDedupKey.values()];
this.#outboundEventsByDedupKey.clear();
let undelivered = false;
for (const peer of this.#webhookPeers) {
try {
await this.#sendWebhookToPeer(peer, events);
} catch (err) {
const peerId = peer.peerId ?? 'unknown';
undelivered = true;
console.warn(
`[broadcast] webhook send to peer ${peerId} failed`,
err,
);
}
}
// A lost flat-perm invalidation has no TTL to heal it.
if (undelivered) this.#requeueOutbound(events);
} finally {
this.#outboundIsFlushing = false;
// Anything that arrived during flush gets the next tick.
@@ -370,6 +376,26 @@ export class BroadcastService extends PuterService {
}
}
/** Put undelivered events back for the next flush, oldest dropped first. */
#requeueOutbound(events: BroadcastEvent[]): void {
for (const event of events) {
const dedupKey = this.#createDedupKey(event);
// Whatever arrived since is fresher — never overwrite it.
if (this.#outboundEventsByDedupKey.has(dedupKey)) continue;
this.#outboundEventsByDedupKey.set(dedupKey, event);
}
let overflow =
this.#outboundEventsByDedupKey.size - this.#outboundMaxQueued;
if (overflow <= 0) return;
console.warn(
`[broadcast] outbound queue over ${this.#outboundMaxQueued}; dropping ${overflow} oldest`,
);
for (const key of this.#outboundEventsByDedupKey.keys()) {
if (overflow-- <= 0) break;
this.#outboundEventsByDedupKey.delete(key);
}
}
async #sendWebhookToPeer(
peer: IBroadcastPeerConfig,
events: BroadcastEvent[],