feat(server): 30s federated-call sentinel worker (TDD)

packages/server/src/utils/federationWorker.ts

@@ -1109,6 +1109,54 @@ async function processHealthCheckTick(): Promise<void> {
   }
 }
 
+// ─── Federated Call Health Sweep ────────────────────────────────────────────
+//
+// Periodic backstop: iterate active FederatedCallEntry objects, look up each
+// distinct host's peer status, and evict entries whose host is non-active.
+// Covers the gap where a peer transitioned to non-active outside of any hook
+// site, or was already non-active when the entry was created.
+//
+// Latency note: real eviction = peer-status-update-lag + tick-period (≤30s).
+// Worst case 15.5min for idle instances with no outbox traffic (health-check
+// worker is the only status source). Documented in the design spec.
+
+export const FEDERATED_CALL_SENTINEL_MS = 30_000;
+
+export async function runFederatedCallSentinelTick(): Promise<void> {
+  const calls = connectionManager.getAllFederatedCalls();
+  if (calls.size === 0) return;
+
+  const distinctHosts = new Set<string>();
+  for (const entry of calls.values()) {
+    distinctHosts.add(entry.federatedCallHost);
+  }
+
+  const db = getDb();
+  for (const host of distinctHosts) {
+    const row = db
+      .select({
+        status: schema.federationPeers.status,
+        instanceName: schema.federationPeers.instanceName,
+      })
+      .from(schema.federationPeers)
+      .where(eq(schema.federationPeers.origin, host))
+      .get();
+
+    if (row && row.status === 'active') continue;
+
+    const isRejectedLike = row?.status === 'rejected' || row?.status === 'revoked';
+    const reason: 'peer_rejected' | 'peer_transient_failure' =
+      isRejectedLike ? 'peer_rejected' : 'peer_transient_failure';
+
+    connectionManager.evictFederatedCallsForHost(host, {
+      reason,
+      peerLabel: row?.instanceName ?? undefined,
+    });
+  }
+}
+
+let federatedCallSentinelTimer: ReturnType<typeof setInterval> | null = null;
+
 // ─── Janitor Worker ──────────────────────────────────────────────────────────
 
 function scheduleJanitorTick(): void {
@@ -1126,6 +1174,11 @@ export function startFederationWorkers(): void {
   scheduleFileQueueTick();
   scheduleHealthCheckTick();
   scheduleJanitorTick();
+  federatedCallSentinelTimer = setInterval(() => {
+    runFederatedCallSentinelTick().catch(err =>
+      console.error('[federation-worker] federatedCallSentinel tick failed:', err)
+    );
+  }, FEDERATED_CALL_SENTINEL_MS);
   // Bootstrap sync for freshly-peered rows (async, non-blocking)
   startupBootstrapSync().catch((err) => {
     console.error('[federation-worker] Startup bootstrap sync error:', err);
@@ -1151,6 +1204,11 @@ export function stopFederationWorkers(): void {
     janitorTimer = null;
   }
 
+  if (federatedCallSentinelTimer) {
+    clearInterval(federatedCallSentinelTimer);
+    federatedCallSentinelTimer = null;
+  }
+
   outboxAbortController?.abort();
   outboxAbortController = null;