fix(federation-worker): auth failures must not increment consecutive_failures

Code review of the previous commit found that the backoff branch of the
new 401/403 handler delegated to handleOutboxDeliveryFailure, which
double-dips by also incrementing consecutive_failures (the network-layer
counter that drives the 'unreachable' transition at threshold 10). Per the
design spec §State Machine Changes → Reset logic, auth failures must
increment consecutive_auth_failures ONLY.

Split handleOutboxDeliveryFailure into:
- applyOutboxEntryBackoff: just the per-entry backoff update (safe to call
  from the auth-failure path)
- handleOutboxDeliveryFailure: entry backoff + peer's consecutive_failures
  bump (network-error path only)

Also adds a console.warn to the backoff branch so operators can diagnose
clock-skew and rotation-grace incidents before the peer hits the terminal
threshold.

Part of backlog #19.
This commit is contained in:
Jannis Braun
2026-04-21 20:45:33 +02:00
parent e5afd376d2
commit 012e489bc7
+22 -7
View File
@@ -5,7 +5,7 @@ import { config } from '../config.js';
import { isFederationRelayEnabled, queueOutboxEvent } from './federationOutbox.js'; import { isFederationRelayEnabled, queueOutboxEvent } from './federationOutbox.js';
import { runFederationJanitor } from './storageJanitor.js'; import { runFederationJanitor } from './storageJanitor.js';
import { buildFederationHeaders, getOurOrigin, generateHmacSecret, ROTATION_GRACE_PERIOD_MS } from './federationAuth.js'; import { buildFederationHeaders, getOurOrigin, generateHmacSecret, ROTATION_GRACE_PERIOD_MS } from './federationAuth.js';
import { evaluateAuthFailure } from './federationAuthFailure.js'; import { evaluateAuthFailure, AUTH_FAILURE_THRESHOLD } from './federationAuthFailure.js';
import { generateSnowflake } from './snowflake.js'; import { generateSnowflake } from './snowflake.js';
import { getDmMessageWithUser } from '../routes/dm.js'; import { getDmMessageWithUser } from '../routes/dm.js';
import { connectionManager } from '../ws/handler.js'; import { connectionManager } from '../ws/handler.js';
@@ -303,7 +303,14 @@ async function processOutboxTick(): Promise<void> {
} }
connectionManager.sendToAdmins({ type: 'federation_peers_changed' as const }); connectionManager.sendToAdmins({ type: 'federation_peers_changed' as const });
} else { } else {
// Below threshold — preserve state, apply backoff to outbox entries // Below threshold — preserve state, apply backoff to outbox entries.
// Do NOT call handleOutboxDeliveryFailure here: per spec, auth failures
// must NOT increment consecutive_failures (that counter drives the
// 'unreachable' transition, which is a network-layer signal, not an
// auth-layer one).
console.warn(
`[federation-worker] Peer ${peerOrigin} returned ${response.status} (auth failure ${decision.newAuthFailures}/${AUTH_FAILURE_THRESHOLD})`,
);
db.update(schema.federationPeers) db.update(schema.federationPeers)
.set({ .set({
consecutiveAuthFailures: decision.newAuthFailures, consecutiveAuthFailures: decision.newAuthFailures,
@@ -311,7 +318,7 @@ async function processOutboxTick(): Promise<void> {
}) })
.where(eq(schema.federationPeers.id, peerId)) .where(eq(schema.federationPeers.id, peerId))
.run(); .run();
handleOutboxDeliveryFailure(db, peerId, peerEntries, now); applyOutboxEntryBackoff(db, peerEntries, now);
} }
} else { } else {
console.warn( console.warn(
@@ -336,13 +343,11 @@ async function processOutboxTick(): Promise<void> {
await resolvePendingPeers(); await resolvePendingPeers();
} }
function handleOutboxDeliveryFailure( function applyOutboxEntryBackoff(
db: ReturnType<typeof getDb>, db: ReturnType<typeof getDb>,
peerId: string,
entries: Array<{ outboxId: string; attempts: number | null }>, entries: Array<{ outboxId: string; attempts: number | null }>,
now: number, now: number,
): void { ): void {
// Increment attempts and compute next retry for each entry
for (const entry of entries) { for (const entry of entries) {
const newAttempts = (entry.attempts ?? 0) + 1; const newAttempts = (entry.attempts ?? 0) + 1;
const backoffMs = getBackoffMs(newAttempts); const backoffMs = getBackoffMs(newAttempts);
@@ -355,8 +360,18 @@ function handleOutboxDeliveryFailure(
.where(eq(schema.federationOutbox.id, entry.outboxId)) .where(eq(schema.federationOutbox.id, entry.outboxId))
.run(); .run();
} }
}
// Update peer failure tracking function handleOutboxDeliveryFailure(
db: ReturnType<typeof getDb>,
peerId: string,
entries: Array<{ outboxId: string; attempts: number | null }>,
now: number,
): void {
applyOutboxEntryBackoff(db, entries, now);
// Update peer failure tracking (network/generic-error path only — auth failures
// use consecutive_auth_failures instead).
const peer = db const peer = db
.select({ consecutiveFailures: schema.federationPeers.consecutiveFailures }) .select({ consecutiveFailures: schema.federationPeers.consecutiveFailures })
.from(schema.federationPeers) .from(schema.federationPeers)