From 0d74d1d112f0114891b906b304c6622ae6e89045 Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Tue, 21 Apr 2026 22:16:09 +0200 Subject: [PATCH] perf(federation-worker): tighten health-check cadence to 15 min HEALTH_CHECK_INTERVAL_MS was 1 h, but ROTATION_GRACE_PERIOD_MS is 15 min. Phase skew between two peers' health-check ticks could stretch rotation finalization desync up to ~1 h, during which signatures from the already- finalized side verify against the other side's primary-only secret (grace has expired; verifyPeerSignature stops trying the pending secret). With AUTH_FAILURE_THRESHOLD = 5 and the existing backoff schedule, this occasionally tripped legitimate rotations into needs_attention. Setting the interval to 15 min (= ROTATION_GRACE_PERIOD_MS) guarantees a finalization tick fires within one grace window on each side, so the cross-verification window where one peer signs with NEW while the other still treats NEW as pending cannot outlast the grace period. Per-tick cost is negligible for the worker's steady state: the only network fetches are per-active-peer /peer/rotate calls when the 90-day rotation interval hits (rare) and per-unreachable-peer /instance/info health pings (bounded by outage count). Going lower than 15 min would reduce the residual desync but increase tick overhead with diminishing returns; 15 min is the grace-period-aligned value that the original spec ("runs hourly") deviated from without justification. Follow-up #20 from S2S DM unification backlog; reduces #19 false-positive rate (outbox auth-failure transition) on legitimate rotations. --- docs/systems/federation.md | 4 ++-- packages/server/src/utils/federationWorker.ts | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/systems/federation.md b/docs/systems/federation.md index de421744..7cac2f84 100644 --- a/docs/systems/federation.md +++ b/docs/systems/federation.md @@ -107,7 +107,7 @@ Both instances store the **same** HMAC secret. The initiating instance generates ### PEER_UNREACHABLE_THRESHOLD -Defined in `federationWorker.ts:45` as `10`. After 10 consecutive delivery failures for a peer, the worker sets `status = 'unreachable'`. The health check worker (1h interval) pings `GET /api/instance/info` on unreachable peers and reverts to `active` on success. +Defined in `federationWorker.ts:47` as `10`. After 10 consecutive delivery failures for a peer, the worker sets `status = 'unreachable'`. The health check worker (15-minute interval, matching `ROTATION_GRACE_PERIOD_MS`) pings `GET /api/instance/info` on unreachable peers and reverts to `active` on success. ### Auto-Peering @@ -1121,7 +1121,7 @@ All workers are started by `startFederationWorkers()` on server boot and stopped |--------|----------|-------|---------|--------| | Outbox delivery | 10s | 50 | 30s | `processOutboxTick` | | File download | 30s | 5 | 60s | `processFileQueueTick` | -| Health check | 1h | all unreachable | 10s | `processHealthCheckTick` | +| Health check | 15min | all unreachable | 10s | `processHealthCheckTick` | | Janitor | 1h | -- | -- | `runFederationJanitor` (sync) | | Initial sync | Once at startup | -- | 30s per page | `runInitialSyncForNewPeers` | diff --git a/packages/server/src/utils/federationWorker.ts b/packages/server/src/utils/federationWorker.ts index 92f26856..d444aa53 100644 --- a/packages/server/src/utils/federationWorker.ts +++ b/packages/server/src/utils/federationWorker.ts @@ -22,7 +22,10 @@ import { Readable } from 'node:stream'; const OUTBOX_INTERVAL_MS = 1_000; // 1 second (idle polls are no-ops) const FILE_QUEUE_INTERVAL_MS = 30_000; // 30 seconds -const HEALTH_CHECK_INTERVAL_MS = 3_600_000; // 1 hour +// Matches ROTATION_GRACE_PERIOD_MS: guarantees a finalization tick fires within +// one grace window on each side, so rotation desync can't outlast the window +// and trip spurious auth failures on the other peer. +const HEALTH_CHECK_INTERVAL_MS = 15 * 60 * 1000; // 15 minutes const JANITOR_INTERVAL_MS = 3_600_000; // 1 hour const OUTBOX_BATCH_LIMIT = 50;