Merge branch 'feat/federation-cleanup-sweep'
This commit is contained in:
@@ -360,7 +360,7 @@ Migration flags (internal): `voice_bit_migrated`, `profile_attachments_cleaned`,
|
|||||||
| status | text NOT NULL | `'active'` | active/pending/awaiting_approval/unreachable/revoked/rejected/needs_attention |
|
| status | text NOT NULL | `'active'` | active/pending/awaiting_approval/unreachable/revoked/rejected/needs_attention |
|
||||||
| lastSeenAt | integer | | |
|
| lastSeenAt | integer | | |
|
||||||
| lastFailureAt | integer | | |
|
| lastFailureAt | integer | | |
|
||||||
| consecutiveFailures | integer | 0 | >=10 → unreachable (network/5xx failures) |
|
| consecutiveFailures | integer NOT NULL | 0 | >=10 → unreachable (network/5xx failures). Counter — never null. |
|
||||||
| consecutiveAuthFailures | integer NOT NULL | 0 | >=5 → needs_attention. Tracked separately from `consecutiveFailures` (network) because auth (401/403) and network failures have different resolution paths. |
|
| consecutiveAuthFailures | integer NOT NULL | 0 | >=5 → needs_attention. Tracked separately from `consecutiveFailures` (network) because auth (401/403) and network failures have different resolution paths. |
|
||||||
| lastSyncedAt | integer | 0 | |
|
| lastSyncedAt | integer | 0 | |
|
||||||
| remoteMaxUploadSize | integer | | Bytes, from peer |
|
| remoteMaxUploadSize | integer | | Bytes, from peer |
|
||||||
|
|||||||
@@ -107,7 +107,7 @@ Both instances store the **same** HMAC secret. The initiating instance generates
|
|||||||
|
|
||||||
### PEER_UNREACHABLE_THRESHOLD
|
### PEER_UNREACHABLE_THRESHOLD
|
||||||
|
|
||||||
Defined in `federationWorker.ts:45` as `10`. After 10 consecutive delivery failures for a peer, the worker sets `status = 'unreachable'`. The health check worker (1h interval) pings `GET /api/instance/info` on unreachable peers and reverts to `active` on success.
|
Defined in `federationWorker.ts:47` as `10`. After 10 consecutive delivery failures for a peer, the worker sets `status = 'unreachable'`. The health check worker (15-minute interval, matching `ROTATION_GRACE_PERIOD_MS`) pings `GET /api/instance/info` on unreachable peers and reverts to `active` on success.
|
||||||
|
|
||||||
### Auto-Peering
|
### Auto-Peering
|
||||||
|
|
||||||
@@ -1121,7 +1121,7 @@ All workers are started by `startFederationWorkers()` on server boot and stopped
|
|||||||
|--------|----------|-------|---------|--------|
|
|--------|----------|-------|---------|--------|
|
||||||
| Outbox delivery | 10s | 50 | 30s | `processOutboxTick` |
|
| Outbox delivery | 10s | 50 | 30s | `processOutboxTick` |
|
||||||
| File download | 30s | 5 | 60s | `processFileQueueTick` |
|
| File download | 30s | 5 | 60s | `processFileQueueTick` |
|
||||||
| Health check | 1h | all unreachable | 10s | `processHealthCheckTick` |
|
| Health check | 15min | all unreachable | 10s | `processHealthCheckTick` |
|
||||||
| Janitor | 1h | -- | -- | `runFederationJanitor` (sync) |
|
| Janitor | 1h | -- | -- | `runFederationJanitor` (sync) |
|
||||||
| Initial sync | Once at startup | -- | 30s per page | `runInitialSyncForNewPeers` |
|
| Initial sync | Once at startup | -- | 30s per page | `runInitialSyncForNewPeers` |
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
PRAGMA defer_foreign_keys=ON;
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE `__new_federation_peers` (
|
||||||
|
`id` text PRIMARY KEY NOT NULL,
|
||||||
|
`origin` text NOT NULL,
|
||||||
|
`instance_name` text,
|
||||||
|
`hmac_secret` text NOT NULL,
|
||||||
|
`status` text DEFAULT 'active' NOT NULL,
|
||||||
|
`last_seen_at` integer,
|
||||||
|
`last_failure_at` integer,
|
||||||
|
`consecutive_failures` integer DEFAULT 0 NOT NULL,
|
||||||
|
`consecutive_auth_failures` integer DEFAULT 0 NOT NULL,
|
||||||
|
`last_synced_at` integer DEFAULT 0,
|
||||||
|
`remote_max_upload_size` integer,
|
||||||
|
`nonce_supported` integer DEFAULT 0 NOT NULL,
|
||||||
|
`pending_hmac_secret` text,
|
||||||
|
`secret_rotation_at` integer,
|
||||||
|
`secret_rotated_at` integer,
|
||||||
|
`auto_rotate_interval_days` integer DEFAULT 90 NOT NULL,
|
||||||
|
`created_at` integer NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
|
INSERT INTO `__new_federation_peers`("id", "origin", "instance_name", "hmac_secret", "status", "last_seen_at", "last_failure_at", "consecutive_failures", "consecutive_auth_failures", "last_synced_at", "remote_max_upload_size", "nonce_supported", "pending_hmac_secret", "secret_rotation_at", "secret_rotated_at", "auto_rotate_interval_days", "created_at") SELECT "id", "origin", "instance_name", "hmac_secret", "status", "last_seen_at", "last_failure_at", COALESCE("consecutive_failures", 0), "consecutive_auth_failures", "last_synced_at", "remote_max_upload_size", "nonce_supported", "pending_hmac_secret", "secret_rotation_at", "secret_rotated_at", "auto_rotate_interval_days", "created_at" FROM `federation_peers`;
|
||||||
|
--> statement-breakpoint
|
||||||
|
DROP TABLE `federation_peers`;
|
||||||
|
--> statement-breakpoint
|
||||||
|
ALTER TABLE `__new_federation_peers` RENAME TO `federation_peers`;
|
||||||
|
--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `federation_peers_origin_unique` ON `federation_peers` (`origin`);
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -29,6 +29,13 @@
|
|||||||
"when": 1776789567488,
|
"when": 1776789567488,
|
||||||
"tag": "0003_classy_loki",
|
"tag": "0003_classy_loki",
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"idx": 4,
|
||||||
|
"version": "6",
|
||||||
|
"when": 1776802580650,
|
||||||
|
"tag": "0004_cooing_black_knight",
|
||||||
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@@ -362,7 +362,7 @@ export const federationPeers = sqliteTable('federation_peers', {
|
|||||||
status: text('status').notNull().default('active'),
|
status: text('status').notNull().default('active'),
|
||||||
lastSeenAt: integer('last_seen_at'),
|
lastSeenAt: integer('last_seen_at'),
|
||||||
lastFailureAt: integer('last_failure_at'),
|
lastFailureAt: integer('last_failure_at'),
|
||||||
consecutiveFailures: integer('consecutive_failures').default(0),
|
consecutiveFailures: integer('consecutive_failures').notNull().default(0),
|
||||||
consecutiveAuthFailures: integer('consecutive_auth_failures').notNull().default(0),
|
consecutiveAuthFailures: integer('consecutive_auth_failures').notNull().default(0),
|
||||||
lastSyncedAt: integer('last_synced_at').default(0),
|
lastSyncedAt: integer('last_synced_at').default(0),
|
||||||
remoteMaxUploadSize: integer('remote_max_upload_size'),
|
remoteMaxUploadSize: integer('remote_max_upload_size'),
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ interface SanitizedPeer {
|
|||||||
status: string;
|
status: string;
|
||||||
lastSeenAt: number | null;
|
lastSeenAt: number | null;
|
||||||
lastFailureAt: number | null;
|
lastFailureAt: number | null;
|
||||||
consecutiveFailures: number | null;
|
consecutiveFailures: number;
|
||||||
lastSyncedAt: number | null;
|
lastSyncedAt: number | null;
|
||||||
createdAt: number;
|
createdAt: number;
|
||||||
rotationInProgress: boolean;
|
rotationInProgress: boolean;
|
||||||
|
|||||||
@@ -22,7 +22,10 @@ import { Readable } from 'node:stream';
|
|||||||
|
|
||||||
const OUTBOX_INTERVAL_MS = 1_000; // 1 second (idle polls are no-ops)
|
const OUTBOX_INTERVAL_MS = 1_000; // 1 second (idle polls are no-ops)
|
||||||
const FILE_QUEUE_INTERVAL_MS = 30_000; // 30 seconds
|
const FILE_QUEUE_INTERVAL_MS = 30_000; // 30 seconds
|
||||||
const HEALTH_CHECK_INTERVAL_MS = 3_600_000; // 1 hour
|
// Matches ROTATION_GRACE_PERIOD_MS: guarantees a finalization tick fires within
|
||||||
|
// one grace window on each side, so rotation desync can't outlast the window
|
||||||
|
// and trip spurious auth failures on the other peer.
|
||||||
|
const HEALTH_CHECK_INTERVAL_MS = 15 * 60 * 1000; // 15 minutes
|
||||||
const JANITOR_INTERVAL_MS = 3_600_000; // 1 hour
|
const JANITOR_INTERVAL_MS = 3_600_000; // 1 hour
|
||||||
|
|
||||||
const OUTBOX_BATCH_LIMIT = 50;
|
const OUTBOX_BATCH_LIMIT = 50;
|
||||||
|
|||||||
@@ -4,8 +4,5 @@
|
|||||||
"outDir": "./dist",
|
"outDir": "./dist",
|
||||||
"rootDir": "./src"
|
"rootDir": "./src"
|
||||||
},
|
},
|
||||||
"include": ["src/**/*"],
|
"include": ["src/**/*"]
|
||||||
"references": [
|
|
||||||
{ "path": "../shared" }
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -52,8 +52,11 @@ import type {
|
|||||||
FederationRegistryEntry,
|
FederationRegistryEntry,
|
||||||
FederationIdentityDeleteRequest,
|
FederationIdentityDeleteRequest,
|
||||||
FederationIdentityDeleteResponse,
|
FederationIdentityDeleteResponse,
|
||||||
|
FederationPeer,
|
||||||
} from '@backspace/shared';
|
} from '@backspace/shared';
|
||||||
|
|
||||||
|
export type { FederationPeer };
|
||||||
|
|
||||||
export class RateLimitError extends Error {
|
export class RateLimitError extends Error {
|
||||||
readonly retryAfter: number;
|
readonly retryAfter: number;
|
||||||
constructor(retryAfter: number) {
|
constructor(retryAfter: number) {
|
||||||
@@ -63,22 +66,6 @@ export class RateLimitError extends Error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface FederationPeer {
|
|
||||||
id: string;
|
|
||||||
origin: string;
|
|
||||||
instanceName: string | null;
|
|
||||||
status: string;
|
|
||||||
lastSeenAt: number | null;
|
|
||||||
lastFailureAt: number | null;
|
|
||||||
consecutiveFailures: number | null;
|
|
||||||
consecutiveAuthFailures: number;
|
|
||||||
lastSyncedAt: number | null;
|
|
||||||
createdAt: number;
|
|
||||||
secretRotatedAt: number | null;
|
|
||||||
rotationInProgress: boolean;
|
|
||||||
autoRotateIntervalDays: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ApprovalRequest {
|
export interface ApprovalRequest {
|
||||||
id: string;
|
id: string;
|
||||||
origin: string;
|
origin: string;
|
||||||
|
|||||||
Reference in New Issue
Block a user