fix(federation): per-event fault isolation in syncPeerMutationLog (#25)
Replace the batch-level processRelayEvents call with a per-event loop wrapped in try/catch. On exception: log event type, messageId, timestamp, peer origin, and the error message; continue to the next event. Previously, a single poison-pill event (e.g., UNIQUE conflict from a malformed relay payload) would throw, be caught by the outer try/catch, and block lastSyncedAt from advancing — causing every future activation to retry the same broken window indefinitely. The final 'replayed N events' log line now reports '(K skipped due to errors)' when K > 0, surfacing the count to operators. Individual event failures are logged via console.error with enough context to debug or replay manually. Trade-off documented in docs/systems/federation.md: forward progress of the sync pipeline takes priority over strict at-least-once delivery. An event that fails to process is lost to the receiver unless replayed manually.
This commit is contained in:
@@ -108,6 +108,7 @@ export async function syncPeerMutationLog(
|
||||
console.log(`[federation] Sync-pull from ${activePeer.origin} (reason=${reason}, since=${activePeer.lastSyncedAt ?? 0})`);
|
||||
|
||||
let totalEvents = 0;
|
||||
let skippedEvents = 0;
|
||||
|
||||
type SyncRequestBody = {
|
||||
sinceTimestamp: number;
|
||||
@@ -133,8 +134,20 @@ export async function syncPeerMutationLog(
|
||||
const data = await resp.json() as { events: FederationRelayEvent[]; hasMore: boolean; checkpoint: number };
|
||||
if (data.events.length === 0) return true;
|
||||
const { processRelayEvents } = await import('../routes/federation.js');
|
||||
await processRelayEvents(data.events, activePeer.origin, activePeer.origin, db);
|
||||
totalEvents += data.events.length;
|
||||
for (const event of data.events) {
|
||||
try {
|
||||
await processRelayEvents([event], activePeer.origin, activePeer.origin, db);
|
||||
totalEvents += 1;
|
||||
} catch (err) {
|
||||
skippedEvents += 1;
|
||||
const errMsg = err instanceof Error ? err.message : String(err);
|
||||
console.error(
|
||||
`[federation] Skipping poison-pill event during sync-pull from ${activePeer.origin}: ` +
|
||||
`eventType=${event.eventType} messageId=${event.messageId} timestamp=${event.timestamp} ` +
|
||||
`error=${errMsg}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
since = data.checkpoint;
|
||||
if (!data.hasMore) return true;
|
||||
}
|
||||
@@ -150,8 +163,9 @@ export async function syncPeerMutationLog(
|
||||
.where(eq(schema.federationPeers.id, activePeer.id))
|
||||
.run();
|
||||
|
||||
if (totalEvents > 0) {
|
||||
console.log(`[federation] Sync-pull from ${activePeer.origin} replayed ${totalEvents} events`);
|
||||
if (totalEvents > 0 || skippedEvents > 0) {
|
||||
const skipSuffix = skippedEvents > 0 ? ` (${skippedEvents} skipped due to errors)` : '';
|
||||
console.log(`[federation] Sync-pull from ${activePeer.origin} replayed ${totalEvents} events${skipSuffix}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[federation] Sync-pull from ${activePeer.origin} failed:`, err);
|
||||
|
||||
Reference in New Issue
Block a user