fix(federation): per-event fault isolation in syncPeerMutationLog (#25)

Replace the batch-level processRelayEvents call with a per-event
loop wrapped in try/catch. On exception: log event type, messageId,
timestamp, peer origin, and the error message; continue to the next
event.

Previously, a single poison-pill event (e.g., UNIQUE conflict from
a malformed relay payload) would throw, be caught by the outer
try/catch, and block lastSyncedAt from advancing — causing every
future activation to retry the same broken window indefinitely.

The final 'replayed N events' log line now reports '(K skipped due
to errors)' when K > 0, surfacing the count to operators. Individual
event failures are logged via console.error with enough context to
debug or replay manually.

Trade-off documented in docs/systems/federation.md: forward progress
of the sync pipeline takes priority over strict at-least-once
delivery. An event that fails to process is lost to the receiver
unless replayed manually.
This commit is contained in:
Jannis Braun
2026-04-22 01:45:14 +02:00
parent 091988c718
commit 15e42a7cc1
3 changed files with 78 additions and 6 deletions
@@ -6,6 +6,7 @@ import path from 'node:path';
import { fileURLToPath } from 'node:url';
import * as schema from '../db/schema.js';
import { eq } from 'drizzle-orm';
import * as federationRouteMock from '../routes/federation.js';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
@@ -245,6 +246,59 @@ describe('syncPeerMutationLog', () => {
expect(bodies[3]?.sinceTimestamp).toBe(100); // profile pass re-seeds from peer.lastSyncedAt
expect(bodies[3]?.contextType).toBe('profile');
});
it('skips a poison-pill event, logs it, and advances past it to process subsequent events', async () => {
const { syncPeerMutationLog } = await import('./federationPeerActivation.js');
testDb.insert(schema.federationPeers).values({
id: 'peer-poison', origin: 'https://peer-poison.example', hmacSecret: 'secret',
status: 'active', lastSyncedAt: 100, createdAt: Date.now(),
}).run();
// Return a single batch of 3 events on the DM pass, then empty on friend + profile passes.
let fetchCall = 0;
vi.spyOn(globalThis, 'fetch').mockImplementation(async () => {
fetchCall++;
if (fetchCall === 1) {
return new Response(JSON.stringify({
events: [
{ eventType: 'create', messageId: 'good-1', timestamp: 200, encryptionVersion: 0 },
{ eventType: 'create', messageId: 'poison', timestamp: 300, encryptionVersion: 0 },
{ eventType: 'create', messageId: 'good-2', timestamp: 400, encryptionVersion: 0 },
],
hasMore: false,
checkpoint: 400,
}), { status: 200 });
}
return new Response(JSON.stringify({ events: [], hasMore: false, checkpoint: 100 }), { status: 200 });
});
// Grab the top-level mock and override implementation per-call:
// good-1: resolves, poison: throws, good-2: resolves.
const processMock = vi.mocked(federationRouteMock.processRelayEvents);
processMock.mockClear();
processMock.mockResolvedValueOnce(undefined as never); // good-1
processMock.mockRejectedValueOnce(new Error('simulated processor failure')); // poison
processMock.mockResolvedValueOnce(undefined as never); // good-2
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
const before = Date.now();
await syncPeerMutationLog('peer-poison', 'health_check_recovery');
// Verify processRelayEvents was called per-event: 3 calls for the 3 DM events.
expect(processMock).toHaveBeenCalledTimes(3);
// Verify error logged for the poison event.
expect(errorSpy).toHaveBeenCalled();
const errorMessages = errorSpy.mock.calls.map(c => String(c[0] ?? ''));
expect(errorMessages.some(m => m.includes('poison'))).toBe(true);
expect(errorMessages.some(m => m.includes('simulated processor failure'))).toBe(true);
// Verify lastSyncedAt advanced despite the poison event (the critical property).
const row = testDb.select().from(schema.federationPeers)
.where(eq(schema.federationPeers.id, 'peer-poison')).get();
expect(row?.lastSyncedAt).toBeGreaterThanOrEqual(before);
});
});
describe('onPeerActivated', () => {