From e5afd376d2cd22bee59cba7cd059dec0d672e41e Mon Sep 17 00:00:00 2001
From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com>
Date: Tue, 21 Apr 2026 20:39:37 +0200
Subject: [PATCH] fix(federation-worker): replace 401/403 wipe-and-rehandshake
 with bounded retry
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The previous handler (commit ce33ccf + its 403 extension) wiped hmac_secret
and reset peer status to 'pending' on any 401/403 from an active peer. This
collapsed three distinct failure modes — transient clock skew, legitimate
split-brain, active MITM attempt — into "silently establish new trust
immediately." The remote's /peer/accept idempotent-200-no-update safeguard
then prevented the re-handshake from actually working, producing a 1-req/sec
loop observed during backlog #16 verification.

New behavior: increment consecutive_auth_failures, apply backoff to outbox
entries. At AUTH_FAILURE_THRESHOLD (5) transition to needs_attention,
preserve hmac_secret, surface delivery-impossible to affected users,
notify admins. Secret is NEVER wiped in response to a network-observed
401/403.

Part of backlog #19.
---
 packages/server/src/utils/federationWorker.ts | 65 ++++++++++++++-----
 1 file changed, 47 insertions(+), 18 deletions(-)

diff --git a/packages/server/src/utils/federationWorker.ts b/packages/server/src/utils/federationWorker.ts
index b63bdf2d..350cccf9 100644
--- a/packages/server/src/utils/federationWorker.ts
+++ b/packages/server/src/utils/federationWorker.ts
@@ -5,6 +5,7 @@ import { config } from '../config.js';
 import { isFederationRelayEnabled, queueOutboxEvent } from './federationOutbox.js';
 import { runFederationJanitor } from './storageJanitor.js';
 import { buildFederationHeaders, getOurOrigin, generateHmacSecret, ROTATION_GRACE_PERIOD_MS } from './federationAuth.js';
+import { evaluateAuthFailure } from './federationAuthFailure.js';
 import { generateSnowflake } from './snowflake.js';
 import { getDmMessageWithUser } from '../routes/dm.js';
 import { connectionManager } from '../ws/handler.js';
@@ -265,25 +266,53 @@ async function processOutboxTick(): Promise<void> {
           .where(eq(schema.federationPeers.id, peerId))
           .run();
       } else if (response.status === 401 || response.status === 403) {
-        // 401 = HMAC verification failed (remote deleted our peer record entirely)
-        // 403 = Peer exists but is not active (remote revoked/rejected us)
-        // Both mean the peer relationship is broken on the remote side. Reset to
-        // 'pending' so resolvePendingPeers() triggers a fresh handshake via
-        // ensurePeered() on the next tick.
-        // Outbox entries are preserved (same peer ID) and will deliver after re-peering.
-        console.warn(
-          `[federation-worker] Peer ${peerOrigin} returned ${response.status} (peer stale) — resetting to pending for re-handshake`,
-        );
-        db.update(schema.federationPeers)
-          .set({
-            status: 'pending',
-            hmacSecret: '',  // Will be regenerated by ensurePeered/performHandshake
-            consecutiveFailures: 0,
-            lastFailureAt: now,
-          })
+        // HMAC rejected or remote's peer row non-active. Do NOT re-handshake
+        // via the unauthenticated /peer/accept path — the remote's
+        // idempotent-200-no-update safeguard would loop forever and, more
+        // importantly, re-handshaking in response to a 401 is not how trust
+        // gets healed. Persistent auth failures transition to
+        // needs_attention; bounded retry (AUTH_FAILURE_THRESHOLD) rides out
+        // transient clock skew and rotation-grace edge races.
+        const currentRow = db
+          .select({ consecutiveAuthFailures: schema.federationPeers.consecutiveAuthFailures })
+          .from(schema.federationPeers)
           .where(eq(schema.federationPeers.id, peerId))
-          .run();
-        connectionManager.sendToAdmins({ type: 'federation_peers_changed' as const });
+          .get();
+        const decision = evaluateAuthFailure(currentRow?.consecutiveAuthFailures ?? 0);
+
+        if (decision.kind === 'transition_to_needs_attention') {
+          db.update(schema.federationPeers)
+            .set({
+              status: 'needs_attention',
+              consecutiveAuthFailures: decision.newAuthFailures,
+              lastFailureAt: now,
+            })
+            .where(eq(schema.federationPeers.id, peerId))
+            .run();
+          console.warn(
+            `[federation-worker] Peer ${peerOrigin} transitioned to needs_attention after ${decision.newAuthFailures} consecutive ${response.status} responses`,
+          );
+
+          const contextMap = buildContextMapForPeer(db, peerId);
+          if (contextMap.size > 0) {
+            pushPeerRejectedEvent(
+              peerOrigin,
+              contextMap,
+              'Federation trust broken — admin must reset peering',
+            );
+          }
+          connectionManager.sendToAdmins({ type: 'federation_peers_changed' as const });
+        } else {
+          // Below threshold — preserve state, apply backoff to outbox entries
+          db.update(schema.federationPeers)
+            .set({
+              consecutiveAuthFailures: decision.newAuthFailures,
+              lastFailureAt: now,
+            })
+            .where(eq(schema.federationPeers.id, peerId))
+            .run();
+          handleOutboxDeliveryFailure(db, peerId, peerEntries, now);
+        }
       } else {
         console.warn(
           `[federation-worker] Peer ${peerOrigin} returned HTTP ${response.status}`,