fix(federation): per-event fault isolation in syncPeerMutationLog (#25)

Replace the batch-level processRelayEvents call with a per-event
loop wrapped in try/catch. On exception: log event type, messageId,
timestamp, peer origin, and the error message; continue to the next
event.

Previously, a single poison-pill event (e.g., UNIQUE conflict from
a malformed relay payload) would throw, be caught by the outer
try/catch, and block lastSyncedAt from advancing — causing every
future activation to retry the same broken window indefinitely.

The final 'replayed N events' log line now reports '(K skipped due
to errors)' when K > 0, surfacing the count to operators. Individual
event failures are logged via console.error with enough context to
debug or replay manually.

Trade-off documented in docs/systems/federation.md: forward progress
of the sync pipeline takes priority over strict at-least-once
delivery. An event that fails to process is lost to the receiver
unless replayed manually.
This commit is contained in:
Jannis Braun
2026-04-22 01:45:14 +02:00
parent 091988c718
commit 15e42a7cc1
3 changed files with 78 additions and 6 deletions
@@ -108,6 +108,7 @@ export async function syncPeerMutationLog(
console.log(`[federation] Sync-pull from ${activePeer.origin} (reason=${reason}, since=${activePeer.lastSyncedAt ?? 0})`);
let totalEvents = 0;
let skippedEvents = 0;
type SyncRequestBody = {
sinceTimestamp: number;
@@ -133,8 +134,20 @@ export async function syncPeerMutationLog(
const data = await resp.json() as { events: FederationRelayEvent[]; hasMore: boolean; checkpoint: number };
if (data.events.length === 0) return true;
const { processRelayEvents } = await import('../routes/federation.js');
await processRelayEvents(data.events, activePeer.origin, activePeer.origin, db);
totalEvents += data.events.length;
for (const event of data.events) {
try {
await processRelayEvents([event], activePeer.origin, activePeer.origin, db);
totalEvents += 1;
} catch (err) {
skippedEvents += 1;
const errMsg = err instanceof Error ? err.message : String(err);
console.error(
`[federation] Skipping poison-pill event during sync-pull from ${activePeer.origin}: ` +
`eventType=${event.eventType} messageId=${event.messageId} timestamp=${event.timestamp} ` +
`error=${errMsg}`,
);
}
}
since = data.checkpoint;
if (!data.hasMore) return true;
}
@@ -150,8 +163,9 @@ export async function syncPeerMutationLog(
.where(eq(schema.federationPeers.id, activePeer.id))
.run();
if (totalEvents > 0) {
console.log(`[federation] Sync-pull from ${activePeer.origin} replayed ${totalEvents} events`);
if (totalEvents > 0 || skippedEvents > 0) {
const skipSuffix = skippedEvents > 0 ? ` (${skippedEvents} skipped due to errors)` : '';
console.log(`[federation] Sync-pull from ${activePeer.origin} replayed ${totalEvents} events${skipSuffix}`);
}
} catch (err) {
console.error(`[federation] Sync-pull from ${activePeer.origin} failed:`, err);