feat(federation): near-instant reset detection — probe epoch at the auth-failure transition + on worker startup
A reset peer reaches needs_attention via the auth-failure path (HMAC desynced by the new incarnation) without passing through unreachable, so the 5s recovery probe never saw it — detection waited up to a full 15-min health-check cycle before 'Re-peer & heal' surfaced. Extract detectResetForPeer() and fire it event-driven at the transition, plus a startup sweep for already-stuck peers. 15-min tick remains the backstop.
This commit is contained in:
@@ -12,7 +12,7 @@ import { connectionManager } from '../ws/handler.js';
|
||||
import { generateThumbnail } from './thumbnail.js';
|
||||
import type { FederationRelayRequest, FederationRelayResponse, FederationRelayEvent } from '@backspace/shared';
|
||||
import { startupBootstrapSync, onPeerDeactivated } from './federationPeerActivation.js';
|
||||
import { probePeerReachable, recoverOrDetectReset, detectResetOnNeedsAttentionPeers } from './federationRecovery.js';
|
||||
import { probePeerReachable, recoverOrDetectReset, detectResetOnNeedsAttentionPeers, detectResetForPeer } from './federationRecovery.js';
|
||||
import { backfillReplicatedProfileAssets } from '../routes/federation.js';
|
||||
import { invokePermanentFailureCallback } from './federationRollback.js';
|
||||
import { refreshPeerEpochs, getInstanceId } from './federationEpoch.js';
|
||||
@@ -347,7 +347,10 @@ export async function processOutboxTick(): Promise<void> {
|
||||
// needs_attention; bounded retry (AUTH_FAILURE_THRESHOLD) rides out
|
||||
// transient clock skew and rotation-grace edge races.
|
||||
const currentRow = db
|
||||
.select({ consecutiveAuthFailures: schema.federationPeers.consecutiveAuthFailures })
|
||||
.select({
|
||||
consecutiveAuthFailures: schema.federationPeers.consecutiveAuthFailures,
|
||||
peerInstanceId: schema.federationPeers.peerInstanceId,
|
||||
})
|
||||
.from(schema.federationPeers)
|
||||
.where(eq(schema.federationPeers.id, peerId))
|
||||
.get();
|
||||
@@ -369,6 +372,23 @@ export async function processOutboxTick(): Promise<void> {
|
||||
`[federation-worker] Peer ${peerOrigin} transitioned to needs_attention after ${decision.newAuthFailures} consecutive ${response.status} responses`,
|
||||
);
|
||||
|
||||
// Event-driven reset detection: a genuinely reset peer reaches
|
||||
// needs_attention via THIS auth-failure path (HMAC desynced by the new
|
||||
// incarnation) without ever passing through `unreachable`, so the
|
||||
// 5-second unreachable-only recovery probe never sees it. Probe its
|
||||
// epoch NOW — the instant the connection is declared broken — instead of
|
||||
// waiting up to a full 15-minute health-check cycle for the backstop
|
||||
// sweep. Detection-only (markPeerReset); never flips back to active.
|
||||
// Fire-and-forget: a probe failure is a benign no-op the 15-min tick
|
||||
// retries, and it must not stall the outbox loop.
|
||||
detectResetForPeer({
|
||||
id: peerId,
|
||||
origin: peerOrigin,
|
||||
peerInstanceId: currentRow?.peerInstanceId ?? null,
|
||||
}).catch(err =>
|
||||
console.error('[federation-worker] reset probe on auth-threshold transition failed:', err)
|
||||
);
|
||||
|
||||
const contextMap = buildContextMapForPeer(db, peerId);
|
||||
if (contextMap.size > 0) {
|
||||
pushPeerRejectedEvent(
|
||||
@@ -1275,6 +1295,16 @@ export function startFederationWorkers(): void {
|
||||
console.error('[federation-worker] Startup bootstrap sync error:', err);
|
||||
});
|
||||
|
||||
// Startup reset-detection sweep: probe every peer already parked in
|
||||
// `needs_attention` for an epoch change. This catches a peer that was reset
|
||||
// while this instance was down (so no live transition fired) AND any peer that
|
||||
// crossed into needs_attention before this build shipped the event-driven
|
||||
// probe — surfacing "Re-peer & heal" immediately on boot instead of on the
|
||||
// next 15-minute health-check cycle. Best-effort, detection-only.
|
||||
detectResetOnNeedsAttentionPeers().catch((err) => {
|
||||
console.error('[federation-worker] Startup reset-detection sweep error:', err);
|
||||
});
|
||||
|
||||
// Backfill any replicated user avatars/banners still stored as absolute URLs
|
||||
// (legacy data from before file replication, or rows whose home was offline
|
||||
// on a previous attempt). Best-effort and idempotent — safe to re-run.
|
||||
|
||||
Reference in New Issue
Block a user