Merge branch 'feat/federation-cleanup-sweep'

This commit is contained in:
Jannis Braun
2026-04-21 22:36:02 +02:00
10 changed files with 3274 additions and 26 deletions
+1 -1
View File
@@ -360,7 +360,7 @@ Migration flags (internal): `voice_bit_migrated`, `profile_attachments_cleaned`,
| status | text NOT NULL | `'active'` | active/pending/awaiting_approval/unreachable/revoked/rejected/needs_attention | | status | text NOT NULL | `'active'` | active/pending/awaiting_approval/unreachable/revoked/rejected/needs_attention |
| lastSeenAt | integer | | | | lastSeenAt | integer | | |
| lastFailureAt | integer | | | | lastFailureAt | integer | | |
| consecutiveFailures | integer | 0 | >=10 → unreachable (network/5xx failures) | | consecutiveFailures | integer NOT NULL | 0 | >=10 → unreachable (network/5xx failures). Counter — never null. |
| consecutiveAuthFailures | integer NOT NULL | 0 | >=5 → needs_attention. Tracked separately from `consecutiveFailures` (network) because auth (401/403) and network failures have different resolution paths. | | consecutiveAuthFailures | integer NOT NULL | 0 | >=5 → needs_attention. Tracked separately from `consecutiveFailures` (network) because auth (401/403) and network failures have different resolution paths. |
| lastSyncedAt | integer | 0 | | | lastSyncedAt | integer | 0 | |
| remoteMaxUploadSize | integer | | Bytes, from peer | | remoteMaxUploadSize | integer | | Bytes, from peer |
+2 -2
View File
@@ -107,7 +107,7 @@ Both instances store the **same** HMAC secret. The initiating instance generates
### PEER_UNREACHABLE_THRESHOLD ### PEER_UNREACHABLE_THRESHOLD
Defined in `federationWorker.ts:45` as `10`. After 10 consecutive delivery failures for a peer, the worker sets `status = 'unreachable'`. The health check worker (1h interval) pings `GET /api/instance/info` on unreachable peers and reverts to `active` on success. Defined in `federationWorker.ts:47` as `10`. After 10 consecutive delivery failures for a peer, the worker sets `status = 'unreachable'`. The health check worker (15-minute interval, matching `ROTATION_GRACE_PERIOD_MS`) pings `GET /api/instance/info` on unreachable peers and reverts to `active` on success.
### Auto-Peering ### Auto-Peering
@@ -1121,7 +1121,7 @@ All workers are started by `startFederationWorkers()` on server boot and stopped
|--------|----------|-------|---------|--------| |--------|----------|-------|---------|--------|
| Outbox delivery | 10s | 50 | 30s | `processOutboxTick` | | Outbox delivery | 10s | 50 | 30s | `processOutboxTick` |
| File download | 30s | 5 | 60s | `processFileQueueTick` | | File download | 30s | 5 | 60s | `processFileQueueTick` |
| Health check | 1h | all unreachable | 10s | `processHealthCheckTick` | | Health check | 15min | all unreachable | 10s | `processHealthCheckTick` |
| Janitor | 1h | -- | -- | `runFederationJanitor` (sync) | | Janitor | 1h | -- | -- | `runFederationJanitor` (sync) |
| Initial sync | Once at startup | -- | 30s per page | `runInitialSyncForNewPeers` | | Initial sync | Once at startup | -- | 30s per page | `runInitialSyncForNewPeers` |
@@ -0,0 +1,29 @@
PRAGMA defer_foreign_keys=ON;
--> statement-breakpoint
CREATE TABLE `__new_federation_peers` (
`id` text PRIMARY KEY NOT NULL,
`origin` text NOT NULL,
`instance_name` text,
`hmac_secret` text NOT NULL,
`status` text DEFAULT 'active' NOT NULL,
`last_seen_at` integer,
`last_failure_at` integer,
`consecutive_failures` integer DEFAULT 0 NOT NULL,
`consecutive_auth_failures` integer DEFAULT 0 NOT NULL,
`last_synced_at` integer DEFAULT 0,
`remote_max_upload_size` integer,
`nonce_supported` integer DEFAULT 0 NOT NULL,
`pending_hmac_secret` text,
`secret_rotation_at` integer,
`secret_rotated_at` integer,
`auto_rotate_interval_days` integer DEFAULT 90 NOT NULL,
`created_at` integer NOT NULL
);
--> statement-breakpoint
INSERT INTO `__new_federation_peers`("id", "origin", "instance_name", "hmac_secret", "status", "last_seen_at", "last_failure_at", "consecutive_failures", "consecutive_auth_failures", "last_synced_at", "remote_max_upload_size", "nonce_supported", "pending_hmac_secret", "secret_rotation_at", "secret_rotated_at", "auto_rotate_interval_days", "created_at") SELECT "id", "origin", "instance_name", "hmac_secret", "status", "last_seen_at", "last_failure_at", COALESCE("consecutive_failures", 0), "consecutive_auth_failures", "last_synced_at", "remote_max_upload_size", "nonce_supported", "pending_hmac_secret", "secret_rotation_at", "secret_rotated_at", "auto_rotate_interval_days", "created_at" FROM `federation_peers`;
--> statement-breakpoint
DROP TABLE `federation_peers`;
--> statement-breakpoint
ALTER TABLE `__new_federation_peers` RENAME TO `federation_peers`;
--> statement-breakpoint
CREATE UNIQUE INDEX `federation_peers_origin_unique` ON `federation_peers` (`origin`);
File diff suppressed because it is too large Load Diff
@@ -29,6 +29,13 @@
"when": 1776789567488, "when": 1776789567488,
"tag": "0003_classy_loki", "tag": "0003_classy_loki",
"breakpoints": true "breakpoints": true
},
{
"idx": 4,
"version": "6",
"when": 1776802580650,
"tag": "0004_cooing_black_knight",
"breakpoints": true
} }
] ]
} }
+1 -1
View File
@@ -362,7 +362,7 @@ export const federationPeers = sqliteTable('federation_peers', {
status: text('status').notNull().default('active'), status: text('status').notNull().default('active'),
lastSeenAt: integer('last_seen_at'), lastSeenAt: integer('last_seen_at'),
lastFailureAt: integer('last_failure_at'), lastFailureAt: integer('last_failure_at'),
consecutiveFailures: integer('consecutive_failures').default(0), consecutiveFailures: integer('consecutive_failures').notNull().default(0),
consecutiveAuthFailures: integer('consecutive_auth_failures').notNull().default(0), consecutiveAuthFailures: integer('consecutive_auth_failures').notNull().default(0),
lastSyncedAt: integer('last_synced_at').default(0), lastSyncedAt: integer('last_synced_at').default(0),
remoteMaxUploadSize: integer('remote_max_upload_size'), remoteMaxUploadSize: integer('remote_max_upload_size'),
+1 -1
View File
@@ -27,7 +27,7 @@ interface SanitizedPeer {
status: string; status: string;
lastSeenAt: number | null; lastSeenAt: number | null;
lastFailureAt: number | null; lastFailureAt: number | null;
consecutiveFailures: number | null; consecutiveFailures: number;
lastSyncedAt: number | null; lastSyncedAt: number | null;
createdAt: number; createdAt: number;
rotationInProgress: boolean; rotationInProgress: boolean;
@@ -22,7 +22,10 @@ import { Readable } from 'node:stream';
const OUTBOX_INTERVAL_MS = 1_000; // 1 second (idle polls are no-ops) const OUTBOX_INTERVAL_MS = 1_000; // 1 second (idle polls are no-ops)
const FILE_QUEUE_INTERVAL_MS = 30_000; // 30 seconds const FILE_QUEUE_INTERVAL_MS = 30_000; // 30 seconds
const HEALTH_CHECK_INTERVAL_MS = 3_600_000; // 1 hour // Matches ROTATION_GRACE_PERIOD_MS: guarantees a finalization tick fires within
// one grace window on each side, so rotation desync can't outlast the window
// and trip spurious auth failures on the other peer.
const HEALTH_CHECK_INTERVAL_MS = 15 * 60 * 1000; // 15 minutes
const JANITOR_INTERVAL_MS = 3_600_000; // 1 hour const JANITOR_INTERVAL_MS = 3_600_000; // 1 hour
const OUTBOX_BATCH_LIMIT = 50; const OUTBOX_BATCH_LIMIT = 50;
+1 -4
View File
@@ -4,8 +4,5 @@
"outDir": "./dist", "outDir": "./dist",
"rootDir": "./src" "rootDir": "./src"
}, },
"include": ["src/**/*"], "include": ["src/**/*"]
"references": [
{ "path": "../shared" }
]
} }
+3 -16
View File
@@ -52,8 +52,11 @@ import type {
FederationRegistryEntry, FederationRegistryEntry,
FederationIdentityDeleteRequest, FederationIdentityDeleteRequest,
FederationIdentityDeleteResponse, FederationIdentityDeleteResponse,
FederationPeer,
} from '@backspace/shared'; } from '@backspace/shared';
export type { FederationPeer };
export class RateLimitError extends Error { export class RateLimitError extends Error {
readonly retryAfter: number; readonly retryAfter: number;
constructor(retryAfter: number) { constructor(retryAfter: number) {
@@ -63,22 +66,6 @@ export class RateLimitError extends Error {
} }
} }
export interface FederationPeer {
id: string;
origin: string;
instanceName: string | null;
status: string;
lastSeenAt: number | null;
lastFailureAt: number | null;
consecutiveFailures: number | null;
consecutiveAuthFailures: number;
lastSyncedAt: number | null;
createdAt: number;
secretRotatedAt: number | null;
rotationInProgress: boolean;
autoRotateIntervalDays: number;
}
export interface ApprovalRequest { export interface ApprovalRequest {
id: string; id: string;
origin: string; origin: string;