diff --git a/src/backend/services/broadcast/BroadcastService.test.ts b/src/backend/services/broadcast/BroadcastService.test.ts index 54f69a1d6..2af9f2acf 100644 --- a/src/backend/services/broadcast/BroadcastService.test.ts +++ b/src/backend/services/broadcast/BroadcastService.test.ts @@ -603,6 +603,33 @@ describe('BroadcastService outbound flush', () => { ); expect(occurrences).toBe(1); }); + + // A dropped invalidation can outlive every TTL meant to heal it. + it('sends a failed event again instead of dropping it', async () => { + const cacheKey = 'requeue-test'; + const carries = (call: unknown) => { + const body = (call as [{ data?: string }])[0]?.data; + return typeof body === 'string' && body.includes(cacheKey); + }; + + axiosRequestMock.mockRejectedValue(new Error('peer unreachable')); + server.clients.event.emit( + 'outer.cacheUpdate' as never, + { cacheKey: [cacheKey] } as never, + {}, + ); + await waitForFlush(() => axiosRequestMock.mock.calls.some(carries)); + + axiosRequestMock.mockReset(); + axiosRequestMock.mockResolvedValue({ + status: 200, + statusText: 'OK', + data: 'ok', + }); + + // Nothing new is emitted — only the retry can satisfy this. + await waitForFlush(() => axiosRequestMock.mock.calls.some(carries)); + }); }); // -- Header validation ------------------------------------------------ diff --git a/src/backend/services/broadcast/BroadcastService.ts b/src/backend/services/broadcast/BroadcastService.ts index b19e7edc7..30b060dcd 100644 --- a/src/backend/services/broadcast/BroadcastService.ts +++ b/src/backend/services/broadcast/BroadcastService.ts @@ -98,6 +98,8 @@ export class BroadcastService extends PuterService { #webhookReplayWindowSeconds = 300; #outboundFlushMs = 2000; + /** Bound on re-queued events, so a peer that stays down can't grow it. */ + #outboundMaxQueued = 10_000; #webhookProtocol: 'http' | 'https' = 'https'; #webhookHostHeader: string | null = null; /** Self-signed certs are common between Puter nodes — accept them. */ @@ -350,17 +352,21 @@ export class BroadcastService extends PuterService { const events = [...this.#outboundEventsByDedupKey.values()]; this.#outboundEventsByDedupKey.clear(); + let undelivered = false; for (const peer of this.#webhookPeers) { try { await this.#sendWebhookToPeer(peer, events); } catch (err) { const peerId = peer.peerId ?? 'unknown'; + undelivered = true; console.warn( `[broadcast] webhook send to peer ${peerId} failed`, err, ); } } + // A lost flat-perm invalidation has no TTL to heal it. + if (undelivered) this.#requeueOutbound(events); } finally { this.#outboundIsFlushing = false; // Anything that arrived during flush gets the next tick. @@ -370,6 +376,26 @@ export class BroadcastService extends PuterService { } } + /** Put undelivered events back for the next flush, oldest dropped first. */ + #requeueOutbound(events: BroadcastEvent[]): void { + for (const event of events) { + const dedupKey = this.#createDedupKey(event); + // Whatever arrived since is fresher — never overwrite it. + if (this.#outboundEventsByDedupKey.has(dedupKey)) continue; + this.#outboundEventsByDedupKey.set(dedupKey, event); + } + let overflow = + this.#outboundEventsByDedupKey.size - this.#outboundMaxQueued; + if (overflow <= 0) return; + console.warn( + `[broadcast] outbound queue over ${this.#outboundMaxQueued}; dropping ${overflow} oldest`, + ); + for (const key of this.#outboundEventsByDedupKey.keys()) { + if (overflow-- <= 0) break; + this.#outboundEventsByDedupKey.delete(key); + } + } + async #sendWebhookToPeer( peer: IBroadcastPeerConfig, events: BroadcastEvent[],