feat(federation): near-instant reset detection — probe epoch at the auth-failure transition + on worker startup

A reset peer reaches needs_attention via the auth-failure path (HMAC desynced by
the new incarnation) without passing through unreachable, so the 5s recovery
probe never saw it — detection waited up to a full 15-min health-check cycle
before 'Re-peer & heal' surfaced. Extract detectResetForPeer() and fire it
event-driven at the transition, plus a startup sweep for already-stuck peers.
15-min tick remains the backstop.
This commit is contained in:
Jannis Braun
2026-07-02 20:10:35 +02:00
parent 13d050c1bb
commit 54ab660204
4 changed files with 138 additions and 10 deletions
+32 -2
View File
@@ -12,7 +12,7 @@ import { connectionManager } from '../ws/handler.js';
import { generateThumbnail } from './thumbnail.js';
import type { FederationRelayRequest, FederationRelayResponse, FederationRelayEvent } from '@backspace/shared';
import { startupBootstrapSync, onPeerDeactivated } from './federationPeerActivation.js';
import { probePeerReachable, recoverOrDetectReset, detectResetOnNeedsAttentionPeers } from './federationRecovery.js';
import { probePeerReachable, recoverOrDetectReset, detectResetOnNeedsAttentionPeers, detectResetForPeer } from './federationRecovery.js';
import { backfillReplicatedProfileAssets } from '../routes/federation.js';
import { invokePermanentFailureCallback } from './federationRollback.js';
import { refreshPeerEpochs, getInstanceId } from './federationEpoch.js';
@@ -347,7 +347,10 @@ export async function processOutboxTick(): Promise<void> {
// needs_attention; bounded retry (AUTH_FAILURE_THRESHOLD) rides out
// transient clock skew and rotation-grace edge races.
const currentRow = db
.select({ consecutiveAuthFailures: schema.federationPeers.consecutiveAuthFailures })
.select({
consecutiveAuthFailures: schema.federationPeers.consecutiveAuthFailures,
peerInstanceId: schema.federationPeers.peerInstanceId,
})
.from(schema.federationPeers)
.where(eq(schema.federationPeers.id, peerId))
.get();
@@ -369,6 +372,23 @@ export async function processOutboxTick(): Promise<void> {
`[federation-worker] Peer ${peerOrigin} transitioned to needs_attention after ${decision.newAuthFailures} consecutive ${response.status} responses`,
);
// Event-driven reset detection: a genuinely reset peer reaches
// needs_attention via THIS auth-failure path (HMAC desynced by the new
// incarnation) without ever passing through `unreachable`, so the
// 5-second unreachable-only recovery probe never sees it. Probe its
// epoch NOW — the instant the connection is declared broken — instead of
// waiting up to a full 15-minute health-check cycle for the backstop
// sweep. Detection-only (markPeerReset); never flips back to active.
// Fire-and-forget: a probe failure is a benign no-op the 15-min tick
// retries, and it must not stall the outbox loop.
detectResetForPeer({
id: peerId,
origin: peerOrigin,
peerInstanceId: currentRow?.peerInstanceId ?? null,
}).catch(err =>
console.error('[federation-worker] reset probe on auth-threshold transition failed:', err)
);
const contextMap = buildContextMapForPeer(db, peerId);
if (contextMap.size > 0) {
pushPeerRejectedEvent(
@@ -1275,6 +1295,16 @@ export function startFederationWorkers(): void {
console.error('[federation-worker] Startup bootstrap sync error:', err);
});
// Startup reset-detection sweep: probe every peer already parked in
// `needs_attention` for an epoch change. This catches a peer that was reset
// while this instance was down (so no live transition fired) AND any peer that
// crossed into needs_attention before this build shipped the event-driven
// probe — surfacing "Re-peer & heal" immediately on boot instead of on the
// next 15-minute health-check cycle. Best-effort, detection-only.
detectResetOnNeedsAttentionPeers().catch((err) => {
console.error('[federation-worker] Startup reset-detection sweep error:', err);
});
// Backfill any replicated user avatars/banners still stored as absolute URLs
// (legacy data from before file replication, or rows whose home was offline
// on a previous attempt). Best-effort and idempotent — safe to re-run.