fix(federation-worker): auth failures must not increment consecutive_failures
Code review of the previous commit found that the backoff branch of the new 401/403 handler delegated to handleOutboxDeliveryFailure, which double-dips by also incrementing consecutive_failures (the network-layer counter that drives the 'unreachable' transition at threshold 10). Per the design spec §State Machine Changes → Reset logic, auth failures must increment consecutive_auth_failures ONLY. Split handleOutboxDeliveryFailure into: - applyOutboxEntryBackoff: just the per-entry backoff update (safe to call from the auth-failure path) - handleOutboxDeliveryFailure: entry backoff + peer's consecutive_failures bump (network-error path only) Also adds a console.warn to the backoff branch so operators can diagnose clock-skew and rotation-grace incidents before the peer hits the terminal threshold. Part of backlog #19.
This commit is contained in:
@@ -5,7 +5,7 @@ import { config } from '../config.js';
|
|||||||
import { isFederationRelayEnabled, queueOutboxEvent } from './federationOutbox.js';
|
import { isFederationRelayEnabled, queueOutboxEvent } from './federationOutbox.js';
|
||||||
import { runFederationJanitor } from './storageJanitor.js';
|
import { runFederationJanitor } from './storageJanitor.js';
|
||||||
import { buildFederationHeaders, getOurOrigin, generateHmacSecret, ROTATION_GRACE_PERIOD_MS } from './federationAuth.js';
|
import { buildFederationHeaders, getOurOrigin, generateHmacSecret, ROTATION_GRACE_PERIOD_MS } from './federationAuth.js';
|
||||||
import { evaluateAuthFailure } from './federationAuthFailure.js';
|
import { evaluateAuthFailure, AUTH_FAILURE_THRESHOLD } from './federationAuthFailure.js';
|
||||||
import { generateSnowflake } from './snowflake.js';
|
import { generateSnowflake } from './snowflake.js';
|
||||||
import { getDmMessageWithUser } from '../routes/dm.js';
|
import { getDmMessageWithUser } from '../routes/dm.js';
|
||||||
import { connectionManager } from '../ws/handler.js';
|
import { connectionManager } from '../ws/handler.js';
|
||||||
@@ -303,7 +303,14 @@ async function processOutboxTick(): Promise<void> {
|
|||||||
}
|
}
|
||||||
connectionManager.sendToAdmins({ type: 'federation_peers_changed' as const });
|
connectionManager.sendToAdmins({ type: 'federation_peers_changed' as const });
|
||||||
} else {
|
} else {
|
||||||
// Below threshold — preserve state, apply backoff to outbox entries
|
// Below threshold — preserve state, apply backoff to outbox entries.
|
||||||
|
// Do NOT call handleOutboxDeliveryFailure here: per spec, auth failures
|
||||||
|
// must NOT increment consecutive_failures (that counter drives the
|
||||||
|
// 'unreachable' transition, which is a network-layer signal, not an
|
||||||
|
// auth-layer one).
|
||||||
|
console.warn(
|
||||||
|
`[federation-worker] Peer ${peerOrigin} returned ${response.status} (auth failure ${decision.newAuthFailures}/${AUTH_FAILURE_THRESHOLD})`,
|
||||||
|
);
|
||||||
db.update(schema.federationPeers)
|
db.update(schema.federationPeers)
|
||||||
.set({
|
.set({
|
||||||
consecutiveAuthFailures: decision.newAuthFailures,
|
consecutiveAuthFailures: decision.newAuthFailures,
|
||||||
@@ -311,7 +318,7 @@ async function processOutboxTick(): Promise<void> {
|
|||||||
})
|
})
|
||||||
.where(eq(schema.federationPeers.id, peerId))
|
.where(eq(schema.federationPeers.id, peerId))
|
||||||
.run();
|
.run();
|
||||||
handleOutboxDeliveryFailure(db, peerId, peerEntries, now);
|
applyOutboxEntryBackoff(db, peerEntries, now);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
console.warn(
|
console.warn(
|
||||||
@@ -336,13 +343,11 @@ async function processOutboxTick(): Promise<void> {
|
|||||||
await resolvePendingPeers();
|
await resolvePendingPeers();
|
||||||
}
|
}
|
||||||
|
|
||||||
function handleOutboxDeliveryFailure(
|
function applyOutboxEntryBackoff(
|
||||||
db: ReturnType<typeof getDb>,
|
db: ReturnType<typeof getDb>,
|
||||||
peerId: string,
|
|
||||||
entries: Array<{ outboxId: string; attempts: number | null }>,
|
entries: Array<{ outboxId: string; attempts: number | null }>,
|
||||||
now: number,
|
now: number,
|
||||||
): void {
|
): void {
|
||||||
// Increment attempts and compute next retry for each entry
|
|
||||||
for (const entry of entries) {
|
for (const entry of entries) {
|
||||||
const newAttempts = (entry.attempts ?? 0) + 1;
|
const newAttempts = (entry.attempts ?? 0) + 1;
|
||||||
const backoffMs = getBackoffMs(newAttempts);
|
const backoffMs = getBackoffMs(newAttempts);
|
||||||
@@ -355,8 +360,18 @@ function handleOutboxDeliveryFailure(
|
|||||||
.where(eq(schema.federationOutbox.id, entry.outboxId))
|
.where(eq(schema.federationOutbox.id, entry.outboxId))
|
||||||
.run();
|
.run();
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Update peer failure tracking
|
function handleOutboxDeliveryFailure(
|
||||||
|
db: ReturnType<typeof getDb>,
|
||||||
|
peerId: string,
|
||||||
|
entries: Array<{ outboxId: string; attempts: number | null }>,
|
||||||
|
now: number,
|
||||||
|
): void {
|
||||||
|
applyOutboxEntryBackoff(db, entries, now);
|
||||||
|
|
||||||
|
// Update peer failure tracking (network/generic-error path only — auth failures
|
||||||
|
// use consecutive_auth_failures instead).
|
||||||
const peer = db
|
const peer = db
|
||||||
.select({ consecutiveFailures: schema.federationPeers.consecutiveFailures })
|
.select({ consecutiveFailures: schema.federationPeers.consecutiveFailures })
|
||||||
.from(schema.federationPeers)
|
.from(schema.federationPeers)
|
||||||
|
|||||||
Reference in New Issue
Block a user