From d8f2b1a9c74fc24fe912d6400784eef97735686d Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Wed, 1 Jul 2026 21:11:34 +0200 Subject: [PATCH 01/12] feat(federation): instance epoch schema + minting --- docs/systems/database.md | 19 + .../drizzle/0008_cute_sebastian_shaw.sql | 16 + .../server/drizzle/meta/0008_snapshot.json | 3789 +++++++++++++++++ packages/server/drizzle/meta/_journal.json | 7 + packages/server/src/db/migrate.test.ts | 30 + packages/server/src/db/migrate.ts | 12 + packages/server/src/db/schema.ts | 19 + 7 files changed, 3892 insertions(+) create mode 100644 packages/server/drizzle/0008_cute_sebastian_shaw.sql create mode 100644 packages/server/drizzle/meta/0008_snapshot.json create mode 100644 packages/server/src/db/migrate.test.ts diff --git a/docs/systems/database.md b/docs/systems/database.md index 1a244143..04cf4c38 100644 --- a/docs/systems/database.md +++ b/docs/systems/database.md @@ -33,6 +33,8 @@ IDs: Snowflake text, permissions: bigint decimal strings | passwordChangedAt | integer | | Token revocation: tokens before this rejected | | showActivity | integer NOT NULL | 1 | Rich presence visibility | | federationRegistryUpdatedAt | integer | 0 | LWW timestamp for federation registry sync | +| federationHealPending | integer | 0 | Instance-epoch self-healing: set when a replicated identity is flagged for re-heal after a peer reset | +| federationHomeOrphaned | integer | 0 | Instance-epoch self-healing: set when this user's home instance was factory-reset and the account could not be re-linked | | createdAt | integer NOT NULL | | Epoch ms | ### spaces @@ -366,6 +368,7 @@ The user INSERT, `usedCount` increment, and redemption row INSERT all run in a s | id | integer PK | 1 | | | instanceName | text | `'Backspace'` | | | workerId | integer | | Snowflake worker ID | +| instanceId | text | | Persistent instance epoch (incarnation UUID). Minted once per DB by `ensureDefaults` and guaranteed non-null after boot. Discriminator that lets peers detect this instance was factory-reset (new DB → new epoch on same origin). See [federation.md → Instance-Epoch Self-Healing]. | | discoveryEnabled | integer NOT NULL | 1 | | | maxBitrateKbps | integer NOT NULL | 20000 | | | minBitrateKbps | integer NOT NULL | 500 | | @@ -407,6 +410,22 @@ The user INSERT, `usedCount` increment, and redemption row INSERT all run in a s | remoteMaxUploadSize | integer | | Bytes, from peer | | createdAt | integer NOT NULL | | | | approvalToken | text | | Single-use 64-hex-char token stored when this row is in `awaiting_approval` (received from remote's 202 response). Verified against the inbound `/peer/accept` `approvalToken` field before promoting to `active`. Cleared (`NULL`) on promotion. See [federation.md → Approval Token Verification](federation.md#approval-token-verification). | +| peerInstanceId | text | | Instance-epoch self-healing: the peer's persistent instance epoch (UUID) as last confirmed. `NULL` until first observed. Compared against `observedPeerInstanceId` to detect a factory-reset peer on the same origin. | +| observedPeerInstanceId | text | | Instance-epoch self-healing: the instance epoch most recently reported by the peer. A mismatch with `peerInstanceId` signals the peer was reset. | +| needsAttentionReason | text | | Instance-epoch self-healing: machine-readable reason a peer was moved to `needs_attention` (e.g. epoch reset detected), for admin surfacing. `NULL` when healthy. | + +### federation_reset_events +Instance-epoch self-healing ledger. One row per origin recording a detected federated-peer reset (same origin, new instance epoch). Upserted when a live epoch change is observed; `resolvedAt` is stamped once stale replicated identities from the dead epoch are healed. + +| Column | Type | Default | Notes | +|--------|------|---------|-------| +| origin | text PK | | Peer origin URL whose epoch changed | +| deadEpoch | text NOT NULL | | The instance epoch that was replaced (now stale) | +| newEpoch | text | | The peer's new instance epoch, once known | +| detectedAt | integer NOT NULL | | Epoch ms the reset was detected | +| resolvedAt | integer | | Epoch ms healing completed; `NULL` while in progress | +| stubCount | integer NOT NULL | 0 | Count of replicated identity stubs affected by the reset | +| orphanedAccountCount | integer NOT NULL | 0 | Count of accounts that could not be re-linked to the new epoch | ### peer_approval_requests Queue of peering requests pending admin review when `autoAcceptPeering` is `false`. Holds **both directions**: inbound rows (remote asked to peer with us) and outbound rows (a local user-initiated `ensurePeered` call gated on this side; see [federation.md → Outbound Peering Gate](federation.md#outbound-peering-gate)). UNIQUE on `(origin, direction)` so the same origin may have at most one row per direction simultaneously. Rows expire after 30 days via janitor cleanup. diff --git a/packages/server/drizzle/0008_cute_sebastian_shaw.sql b/packages/server/drizzle/0008_cute_sebastian_shaw.sql new file mode 100644 index 00000000..55b8b5d0 --- /dev/null +++ b/packages/server/drizzle/0008_cute_sebastian_shaw.sql @@ -0,0 +1,16 @@ +CREATE TABLE `federation_reset_events` ( + `origin` text PRIMARY KEY NOT NULL, + `dead_epoch` text NOT NULL, + `new_epoch` text, + `detected_at` integer NOT NULL, + `resolved_at` integer, + `stub_count` integer DEFAULT 0 NOT NULL, + `orphaned_account_count` integer DEFAULT 0 NOT NULL +); +--> statement-breakpoint +ALTER TABLE `federation_peers` ADD `peer_instance_id` text;--> statement-breakpoint +ALTER TABLE `federation_peers` ADD `observed_peer_instance_id` text;--> statement-breakpoint +ALTER TABLE `federation_peers` ADD `needs_attention_reason` text;--> statement-breakpoint +ALTER TABLE `instance_settings` ADD `instance_id` text;--> statement-breakpoint +ALTER TABLE `users` ADD `federation_heal_pending` integer DEFAULT 0;--> statement-breakpoint +ALTER TABLE `users` ADD `federation_home_orphaned` integer DEFAULT 0; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0008_snapshot.json b/packages/server/drizzle/meta/0008_snapshot.json new file mode 100644 index 00000000..84b6461c --- /dev/null +++ b/packages/server/drizzle/meta/0008_snapshot.json @@ -0,0 +1,3789 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "969e7ead-0aa6-44de-939e-ce55337597be", + "prevId": "f682f910-f757-499f-b448-72e51f506b83", + "tables": { + "attachments": { + "name": "attachments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dm_message_id": { + "name": "dm_message_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "uploader_id": { + "name": "uploader_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "original_name": { + "name": "original_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mimetype": { + "name": "mimetype", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thumbnail_filename": { + "name": "thumbnail_filename", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "playable": { + "name": "playable", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "federation_status": { + "name": "federation_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "federation_meta": { + "name": "federation_meta", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_attachments_message_id": { + "name": "idx_attachments_message_id", + "columns": [ + "message_id" + ], + "isUnique": false + }, + "idx_attachments_dm_message_id": { + "name": "idx_attachments_dm_message_id", + "columns": [ + "dm_message_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "attachments_message_id_messages_id_fk": { + "name": "attachments_message_id_messages_id_fk", + "tableFrom": "attachments", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "attachments_dm_message_id_dm_messages_id_fk": { + "name": "attachments_dm_message_id_dm_messages_id_fk", + "tableFrom": "attachments", + "tableTo": "dm_messages", + "columnsFrom": [ + "dm_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "bans": { + "name": "bans", + "columns": { + "space_id": { + "name": "space_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "banned_by": { + "name": "banned_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_bans_space_id": { + "name": "idx_bans_space_id", + "columns": [ + "space_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "bans_space_id_spaces_id_fk": { + "name": "bans_space_id_spaces_id_fk", + "tableFrom": "bans", + "tableTo": "spaces", + "columnsFrom": [ + "space_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "bans_user_id_users_id_fk": { + "name": "bans_user_id_users_id_fk", + "tableFrom": "bans", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "bans_banned_by_users_id_fk": { + "name": "bans_banned_by_users_id_fk", + "tableFrom": "bans", + "tableTo": "users", + "columnsFrom": [ + "banned_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "bans_space_id_user_id_pk": { + "columns": [ + "space_id", + "user_id" + ], + "name": "bans_space_id_user_id_pk" + } + }, + "uniqueConstraints": {} + }, + "category_overrides": { + "name": "category_overrides", + "columns": { + "category_id": { + "name": "category_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "allow": { + "name": "allow", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'0'" + }, + "deny": { + "name": "deny", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'0'" + } + }, + "indexes": { + "idx_category_overrides_category_id": { + "name": "idx_category_overrides_category_id", + "columns": [ + "category_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "category_overrides_category_id_channel_categories_id_fk": { + "name": "category_overrides_category_id_channel_categories_id_fk", + "tableFrom": "category_overrides", + "tableTo": "channel_categories", + "columnsFrom": [ + "category_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "category_overrides_category_id_target_type_target_id_pk": { + "columns": [ + "category_id", + "target_type", + "target_id" + ], + "name": "category_overrides_category_id_target_type_target_id_pk" + } + }, + "uniqueConstraints": {} + }, + "channel_categories": { + "name": "channel_categories", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "space_id": { + "name": "space_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "position": { + "name": "position", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_channel_categories_space_id": { + "name": "idx_channel_categories_space_id", + "columns": [ + "space_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "channel_categories_space_id_spaces_id_fk": { + "name": "channel_categories_space_id_spaces_id_fk", + "tableFrom": "channel_categories", + "tableTo": "spaces", + "columnsFrom": [ + "space_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "channel_overrides": { + "name": "channel_overrides", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "allow": { + "name": "allow", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'0'" + }, + "deny": { + "name": "deny", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'0'" + } + }, + "indexes": { + "idx_channel_overrides_channel_id": { + "name": "idx_channel_overrides_channel_id", + "columns": [ + "channel_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "channel_overrides_channel_id_channels_id_fk": { + "name": "channel_overrides_channel_id_channels_id_fk", + "tableFrom": "channel_overrides", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "channel_overrides_channel_id_target_type_target_id_pk": { + "columns": [ + "channel_id", + "target_type", + "target_id" + ], + "name": "channel_overrides_channel_id_target_type_target_id_pk" + } + }, + "uniqueConstraints": {} + }, + "channels": { + "name": "channels", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "space_id": { + "name": "space_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "topic": { + "name": "topic", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "position": { + "name": "position", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "category_id": { + "name": "category_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_channels_space_id": { + "name": "idx_channels_space_id", + "columns": [ + "space_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "channels_space_id_spaces_id_fk": { + "name": "channels_space_id_spaces_id_fk", + "tableFrom": "channels", + "tableTo": "spaces", + "columnsFrom": [ + "space_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "dm_channels": { + "name": "dm_channels", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "federated_id": { + "name": "federated_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "owner_home_user_id": { + "name": "owner_home_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "owner_home_instance": { + "name": "owner_home_instance", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata_updated_at": { + "name": "metadata_updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_dm_federated": { + "name": "idx_dm_federated", + "columns": [ + "federated_id" + ], + "isUnique": true, + "where": "federated_id IS NOT NULL" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "dm_members": { + "name": "dm_members", + "columns": { + "dm_channel_id": { + "name": "dm_channel_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "closed": { + "name": "closed", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_dm_members_user_id": { + "name": "idx_dm_members_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "dm_members_dm_channel_id_dm_channels_id_fk": { + "name": "dm_members_dm_channel_id_dm_channels_id_fk", + "tableFrom": "dm_members", + "tableTo": "dm_channels", + "columnsFrom": [ + "dm_channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "dm_members_user_id_users_id_fk": { + "name": "dm_members_user_id_users_id_fk", + "tableFrom": "dm_members", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "dm_members_dm_channel_id_user_id_pk": { + "columns": [ + "dm_channel_id", + "user_id" + ], + "name": "dm_members_dm_channel_id_user_id_pk" + } + }, + "uniqueConstraints": {} + }, + "dm_messages": { + "name": "dm_messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "dm_channel_id": { + "name": "dm_channel_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reply_to_id": { + "name": "reply_to_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'user'" + }, + "edited_at": { + "name": "edited_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_instance": { + "name": "source_instance", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_message_id": { + "name": "source_message_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "encryption_version": { + "name": "encryption_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_dm_messages_dm_channel_id": { + "name": "idx_dm_messages_dm_channel_id", + "columns": [ + "dm_channel_id" + ], + "isUnique": false + }, + "idx_dm_messages_user_id": { + "name": "idx_dm_messages_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_dm_messages_source_unique": { + "name": "idx_dm_messages_source_unique", + "columns": [ + "source_instance", + "source_message_id" + ], + "isUnique": true, + "where": "source_instance IS NOT NULL" + } + }, + "foreignKeys": { + "dm_messages_dm_channel_id_dm_channels_id_fk": { + "name": "dm_messages_dm_channel_id_dm_channels_id_fk", + "tableFrom": "dm_messages", + "tableTo": "dm_channels", + "columnsFrom": [ + "dm_channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "dm_messages_user_id_users_id_fk": { + "name": "dm_messages_user_id_users_id_fk", + "tableFrom": "dm_messages", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "dm_messages_reply_to_id_dm_messages_id_fk": { + "name": "dm_messages_reply_to_id_dm_messages_id_fk", + "tableFrom": "dm_messages", + "tableTo": "dm_messages", + "columnsFrom": [ + "reply_to_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "dm_reactions": { + "name": "dm_reactions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "dm_message_id": { + "name": "dm_message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emoji": { + "name": "emoji", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_dm_reactions_dm_message_id": { + "name": "idx_dm_reactions_dm_message_id", + "columns": [ + "dm_message_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "dm_reactions_dm_message_id_dm_messages_id_fk": { + "name": "dm_reactions_dm_message_id_dm_messages_id_fk", + "tableFrom": "dm_reactions", + "tableTo": "dm_messages", + "columnsFrom": [ + "dm_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "dm_reactions_user_id_users_id_fk": { + "name": "dm_reactions_user_id_users_id_fk", + "tableFrom": "dm_reactions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "embeds": { + "name": "embeds", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dm_message_id": { + "name": "dm_message_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "embed_type": { + "name": "embed_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "embed_url": { + "name": "embed_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_embeds_message_id": { + "name": "idx_embeds_message_id", + "columns": [ + "message_id" + ], + "isUnique": false + }, + "idx_embeds_dm_message_id": { + "name": "idx_embeds_dm_message_id", + "columns": [ + "dm_message_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "embeds_message_id_messages_id_fk": { + "name": "embeds_message_id_messages_id_fk", + "tableFrom": "embeds", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "embeds_dm_message_id_dm_messages_id_fk": { + "name": "embeds_dm_message_id_dm_messages_id_fk", + "tableFrom": "embeds", + "tableTo": "dm_messages", + "columnsFrom": [ + "dm_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "federation_file_queue": { + "name": "federation_file_queue", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "peer_origin": { + "name": "peer_origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dm_message_id": { + "name": "dm_message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_filename": { + "name": "target_filename", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "original_name": { + "name": "original_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mimetype": { + "name": "mimetype", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "rejection_reason": { + "name": "rejection_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "next_retry_at": { + "name": "next_retry_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "federation_mutation_log": { + "name": "federation_mutation_log", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "entity_id": { + "name": "entity_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "context_id": { + "name": "context_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "context_type": { + "name": "context_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'dm'" + }, + "mutation_type": { + "name": "mutation_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mutated_at": { + "name": "mutated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_mutation_log_time": { + "name": "idx_mutation_log_time", + "columns": [ + "mutated_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "federation_outbox": { + "name": "federation_outbox", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "peer_id": { + "name": "peer_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "context_id": { + "name": "context_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entity_id": { + "name": "entity_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "context_type": { + "name": "context_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'dm'" + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryption_version": { + "name": "encryption_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "next_retry_at": { + "name": "next_retry_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_outbox_retry": { + "name": "idx_outbox_retry", + "columns": [ + "next_retry_at" + ], + "isUnique": false + }, + "federation_outbox_peer_id_entity_id_unique": { + "name": "federation_outbox_peer_id_entity_id_unique", + "columns": [ + "peer_id", + "entity_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "federation_outbox_peer_id_federation_peers_id_fk": { + "name": "federation_outbox_peer_id_federation_peers_id_fk", + "tableFrom": "federation_outbox", + "tableTo": "federation_peers", + "columnsFrom": [ + "peer_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "federation_peers": { + "name": "federation_peers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "instance_name": { + "name": "instance_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hmac_secret": { + "name": "hmac_secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_failure_at": { + "name": "last_failure_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "consecutive_auth_failures": { + "name": "consecutive_auth_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_probe_at": { + "name": "last_probe_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "probe_attempts": { + "name": "probe_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_synced_at": { + "name": "last_synced_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "remote_max_upload_size": { + "name": "remote_max_upload_size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "nonce_supported": { + "name": "nonce_supported", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "pending_hmac_secret": { + "name": "pending_hmac_secret", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "secret_rotation_at": { + "name": "secret_rotation_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "secret_rotated_at": { + "name": "secret_rotated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "auto_rotate_interval_days": { + "name": "auto_rotate_interval_days", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 90 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "approval_token": { + "name": "approval_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "peer_instance_id": { + "name": "peer_instance_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "observed_peer_instance_id": { + "name": "observed_peer_instance_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "needs_attention_reason": { + "name": "needs_attention_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "federation_peers_origin_unique": { + "name": "federation_peers_origin_unique", + "columns": [ + "origin" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "federation_reset_events": { + "name": "federation_reset_events", + "columns": { + "origin": { + "name": "origin", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "dead_epoch": { + "name": "dead_epoch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "new_epoch": { + "name": "new_epoch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "detected_at": { + "name": "detected_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stub_count": { + "name": "stub_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "orphaned_account_count": { + "name": "orphaned_account_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "friend_requests": { + "name": "friend_requests", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "from_id": { + "name": "from_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "to_id": { + "name": "to_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "relay_message_id": { + "name": "relay_message_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_friend_requests_to_id": { + "name": "idx_friend_requests_to_id", + "columns": [ + "to_id" + ], + "isUnique": false + }, + "idx_friend_requests_from_id": { + "name": "idx_friend_requests_from_id", + "columns": [ + "from_id" + ], + "isUnique": false + }, + "idx_friend_requests_relay_message_id": { + "name": "idx_friend_requests_relay_message_id", + "columns": [ + "relay_message_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "friend_requests_from_id_users_id_fk": { + "name": "friend_requests_from_id_users_id_fk", + "tableFrom": "friend_requests", + "tableTo": "users", + "columnsFrom": [ + "from_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "friend_requests_to_id_users_id_fk": { + "name": "friend_requests_to_id_users_id_fk", + "tableFrom": "friend_requests", + "tableTo": "users", + "columnsFrom": [ + "to_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "friends": { + "name": "friends", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "friend_id": { + "name": "friend_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_friends_user_id": { + "name": "idx_friends_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "idx_friends_friend_id": { + "name": "idx_friends_friend_id", + "columns": [ + "friend_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "friends_user_id_users_id_fk": { + "name": "friends_user_id_users_id_fk", + "tableFrom": "friends", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "friends_friend_id_users_id_fk": { + "name": "friends_friend_id_users_id_fk", + "tableFrom": "friends", + "tableTo": "users", + "columnsFrom": [ + "friend_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "friends_user_id_friend_id_pk": { + "columns": [ + "user_id", + "friend_id" + ], + "name": "friends_user_id_friend_id_pk" + } + }, + "uniqueConstraints": {} + }, + "instance_settings": { + "name": "instance_settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "instance_name": { + "name": "instance_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'Backspace'" + }, + "worker_id": { + "name": "worker_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "discovery_enabled": { + "name": "discovery_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "max_bitrate_kbps": { + "name": "max_bitrate_kbps", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 20000 + }, + "min_bitrate_kbps": { + "name": "min_bitrate_kbps", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 500 + }, + "bitrate_step_kbps": { + "name": "bitrate_step_kbps", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 500 + }, + "allowed_resolutions": { + "name": "allowed_resolutions", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'540,720,1080'" + }, + "allowed_framerates": { + "name": "allowed_framerates", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'30,45,60'" + }, + "max_resolution": { + "name": "max_resolution", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1080 + }, + "max_framerate": { + "name": "max_framerate", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 60 + }, + "registration_open": { + "name": "registration_open", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "federated_registration_open": { + "name": "federated_registration_open", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "gif_api_key": { + "name": "gif_api_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bitrate_matrix_overrides": { + "name": "bitrate_matrix_overrides", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "allow_custom_bitrate": { + "name": "allow_custom_bitrate", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "max_upload_size_bytes": { + "name": "max_upload_size_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "federation_relay_enabled": { + "name": "federation_relay_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "federation_relay_ttl_days": { + "name": "federation_relay_ttl_days", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 30 + }, + "default_auto_rotate_interval_days": { + "name": "default_auto_rotate_interval_days", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 90 + }, + "auto_accept_peering": { + "name": "auto_accept_peering", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "invite_links": { + "name": "invite_links", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "max_uses": { + "name": "max_uses", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "used_count": { + "name": "used_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "invite_links_token_unique": { + "name": "invite_links_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "idx_invite_links_created_at": { + "name": "idx_invite_links_created_at", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "invite_links_created_by_users_id_fk": { + "name": "invite_links_created_by_users_id_fk", + "tableFrom": "invite_links", + "tableTo": "users", + "columnsFrom": [ + "created_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "invite_redemptions": { + "name": "invite_redemptions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "invite_id": { + "name": "invite_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "registrant_username": { + "name": "registrant_username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redeemed_at": { + "name": "redeemed_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_invite_redemptions_invite_id": { + "name": "idx_invite_redemptions_invite_id", + "columns": [ + "invite_id" + ], + "isUnique": false + }, + "idx_invite_redemptions_user_id": { + "name": "idx_invite_redemptions_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "invite_redemptions_invite_id_invite_links_id_fk": { + "name": "invite_redemptions_invite_id_invite_links_id_fk", + "tableFrom": "invite_redemptions", + "tableTo": "invite_links", + "columnsFrom": [ + "invite_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invite_redemptions_user_id_users_id_fk": { + "name": "invite_redemptions_user_id_users_id_fk", + "tableFrom": "invite_redemptions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "join_requests": { + "name": "join_requests", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "space_id": { + "name": "space_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "decided_by": { + "name": "decided_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "decided_at": { + "name": "decided_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_join_requests_space_id_status": { + "name": "idx_join_requests_space_id_status", + "columns": [ + "space_id", + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "join_requests_space_id_spaces_id_fk": { + "name": "join_requests_space_id_spaces_id_fk", + "tableFrom": "join_requests", + "tableTo": "spaces", + "columnsFrom": [ + "space_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "join_requests_user_id_users_id_fk": { + "name": "join_requests_user_id_users_id_fk", + "tableFrom": "join_requests", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "join_requests_decided_by_users_id_fk": { + "name": "join_requests_decided_by_users_id_fk", + "tableFrom": "join_requests", + "tableTo": "users", + "columnsFrom": [ + "decided_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "member_roles": { + "name": "member_roles", + "columns": { + "space_id": { + "name": "space_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role_id": { + "name": "role_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_member_roles_user_id_space_id": { + "name": "idx_member_roles_user_id_space_id", + "columns": [ + "user_id", + "space_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "member_roles_space_id_spaces_id_fk": { + "name": "member_roles_space_id_spaces_id_fk", + "tableFrom": "member_roles", + "tableTo": "spaces", + "columnsFrom": [ + "space_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_roles_user_id_users_id_fk": { + "name": "member_roles_user_id_users_id_fk", + "tableFrom": "member_roles", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_roles_role_id_roles_id_fk": { + "name": "member_roles_role_id_roles_id_fk", + "tableFrom": "member_roles", + "tableTo": "roles", + "columnsFrom": [ + "role_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "member_roles_space_id_user_id_role_id_pk": { + "columns": [ + "space_id", + "user_id", + "role_id" + ], + "name": "member_roles_space_id_user_id_role_id_pk" + } + }, + "uniqueConstraints": {} + }, + "messages": { + "name": "messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reply_to_id": { + "name": "reply_to_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "edited_at": { + "name": "edited_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_messages_channel_id": { + "name": "idx_messages_channel_id", + "columns": [ + "channel_id" + ], + "isUnique": false + }, + "idx_messages_user_id": { + "name": "idx_messages_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "messages_channel_id_channels_id_fk": { + "name": "messages_channel_id_channels_id_fk", + "tableFrom": "messages", + "tableTo": "channels", + "columnsFrom": [ + "channel_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "messages_user_id_users_id_fk": { + "name": "messages_user_id_users_id_fk", + "tableFrom": "messages", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "messages_reply_to_id_messages_id_fk": { + "name": "messages_reply_to_id_messages_id_fk", + "tableFrom": "messages", + "tableTo": "messages", + "columnsFrom": [ + "reply_to_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "peer_approval_notifications": { + "name": "peer_approval_notifications", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "peer_origin": { + "name": "peer_origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "trigger_reason": { + "name": "trigger_reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "trigger_target": { + "name": "trigger_target", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "read_at": { + "name": "read_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_peer_approval_notifications_user_id": { + "name": "idx_peer_approval_notifications_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "peer_approval_notifications_user_id_users_id_fk": { + "name": "peer_approval_notifications_user_id_users_id_fk", + "tableFrom": "peer_approval_notifications", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "peer_approval_requests": { + "name": "peer_approval_requests", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "direction": { + "name": "direction", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'inbound'" + }, + "instance_name": { + "name": "instance_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hmac_secret": { + "name": "hmac_secret", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "requested_at": { + "name": "requested_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "approval_token": { + "name": "approval_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "peer_approval_requests_origin_direction_unique": { + "name": "peer_approval_requests_origin_direction_unique", + "columns": [ + "origin", + "direction" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "peer_approval_subscribers": { + "name": "peer_approval_subscribers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "trigger_reason": { + "name": "trigger_reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "trigger_target": { + "name": "trigger_target", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_peer_approval_subscribers_user_id": { + "name": "idx_peer_approval_subscribers_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "peer_approval_subscribers_request_id_user_id_trigger_reason_trigger_target_unique": { + "name": "peer_approval_subscribers_request_id_user_id_trigger_reason_trigger_target_unique", + "columns": [ + "request_id", + "user_id", + "trigger_reason", + "trigger_target" + ], + "isUnique": true + } + }, + "foreignKeys": { + "peer_approval_subscribers_request_id_peer_approval_requests_id_fk": { + "name": "peer_approval_subscribers_request_id_peer_approval_requests_id_fk", + "tableFrom": "peer_approval_subscribers", + "tableTo": "peer_approval_requests", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "peer_approval_subscribers_user_id_users_id_fk": { + "name": "peer_approval_subscribers_user_id_users_id_fk", + "tableFrom": "peer_approval_subscribers", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "reactions": { + "name": "reactions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emoji": { + "name": "emoji", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_reactions_message_id": { + "name": "idx_reactions_message_id", + "columns": [ + "message_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "reactions_message_id_messages_id_fk": { + "name": "reactions_message_id_messages_id_fk", + "tableFrom": "reactions", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "reactions_user_id_users_id_fk": { + "name": "reactions_user_id_users_id_fk", + "tableFrom": "reactions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "read_states": { + "name": "read_states", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_read_message_id": { + "name": "last_read_message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_read_states_user_id": { + "name": "idx_read_states_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "read_states_user_id_users_id_fk": { + "name": "read_states_user_id_users_id_fk", + "tableFrom": "read_states", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "read_states_user_id_channel_id_pk": { + "columns": [ + "user_id", + "channel_id" + ], + "name": "read_states_user_id_channel_id_pk" + } + }, + "uniqueConstraints": {} + }, + "roles": { + "name": "roles", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "space_id": { + "name": "space_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'#b9bbbe'" + }, + "position": { + "name": "position", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_roles_space_id": { + "name": "idx_roles_space_id", + "columns": [ + "space_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "roles_space_id_spaces_id_fk": { + "name": "roles_space_id_spaces_id_fk", + "tableFrom": "roles", + "tableTo": "spaces", + "columnsFrom": [ + "space_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "space_folder_members": { + "name": "space_folder_members", + "columns": { + "folder_id": { + "name": "folder_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "space_id": { + "name": "space_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "position": { + "name": "position", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": { + "space_folder_members_folder_id_space_folders_id_fk": { + "name": "space_folder_members_folder_id_space_folders_id_fk", + "tableFrom": "space_folder_members", + "tableTo": "space_folders", + "columnsFrom": [ + "folder_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "space_folder_members_folder_id_space_id_pk": { + "columns": [ + "folder_id", + "space_id" + ], + "name": "space_folder_members_folder_id_space_id_pk" + } + }, + "uniqueConstraints": {} + }, + "space_folders": { + "name": "space_folders", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "position": { + "name": "position", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "space_folders_user_id_users_id_fk": { + "name": "space_folders_user_id_users_id_fk", + "tableFrom": "space_folders", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "space_members": { + "name": "space_members", + "columns": { + "space_id": { + "name": "space_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "nickname": { + "name": "nickname", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "joined_at": { + "name": "joined_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_space_members_user_id": { + "name": "idx_space_members_user_id", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "space_members_space_id_spaces_id_fk": { + "name": "space_members_space_id_spaces_id_fk", + "tableFrom": "space_members", + "tableTo": "spaces", + "columnsFrom": [ + "space_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "space_members_user_id_users_id_fk": { + "name": "space_members_user_id_users_id_fk", + "tableFrom": "space_members", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "space_members_space_id_user_id_pk": { + "columns": [ + "space_id", + "user_id" + ], + "name": "space_members_space_id_user_id_pk" + } + }, + "uniqueConstraints": {} + }, + "spaces": { + "name": "spaces", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "banner": { + "name": "banner", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avatar_color": { + "name": "avatar_color", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invite_code": { + "name": "invite_code", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "visibility": { + "name": "visibility", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'private'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "spaces_invite_code_unique": { + "name": "spaces_invite_code_unique", + "columns": [ + "invite_code" + ], + "isUnique": true + } + }, + "foreignKeys": { + "spaces_owner_id_users_id_fk": { + "name": "spaces_owner_id_users_id_fk", + "tableFrom": "spaces", + "tableTo": "users", + "columnsFrom": [ + "owner_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "user_federation_registry": { + "name": "user_federation_registry", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "remote_user_id": { + "name": "remote_user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'connected'" + }, + "added_at": { + "name": "added_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_connected_at": { + "name": "last_connected_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disconnected_at": { + "name": "disconnected_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "user_federation_registry_user_id_users_id_fk": { + "name": "user_federation_registry_user_id_users_id_fk", + "tableFrom": "user_federation_registry", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_federation_registry_user_id_origin_pk": { + "columns": [ + "user_id", + "origin" + ], + "name": "user_federation_registry_user_id_origin_pk" + } + }, + "uniqueConstraints": {} + }, + "user_space_layout": { + "name": "user_space_layout", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "layout": { + "name": "layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "user_space_layout_user_id_users_id_fk": { + "name": "user_space_layout_user_id_users_id_fk", + "tableFrom": "user_space_layout", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "avatar": { + "name": "avatar", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'offline'" + }, + "custom_status": { + "name": "custom_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_admin": { + "name": "is_admin", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "home_instance": { + "name": "home_instance", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "home_user_id": { + "name": "home_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "replicated_instances": { + "name": "replicated_instances", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'[]'" + }, + "banner": { + "name": "banner", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "accent_color": { + "name": "accent_color", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avatar_color": { + "name": "avatar_color", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_deleted": { + "name": "is_deleted", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "discoverable": { + "name": "discoverable", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 1 + }, + "profile_updated_at": { + "name": "profile_updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password_changed_at": { + "name": "password_changed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "show_activity": { + "name": "show_activity", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "federation_registry_updated_at": { + "name": "federation_registry_updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "federation_heal_pending": { + "name": "federation_heal_pending", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "federation_home_orphaned": { + "name": "federation_home_orphaned", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "users_username_unique": { + "name": "users_username_unique", + "columns": [ + "username" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "voice_restrictions": { + "name": "voice_restrictions", + "columns": { + "space_id": { + "name": "space_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "restriction_type": { + "name": "restriction_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "moderator_id": { + "name": "moderator_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_voice_restrictions_space_id": { + "name": "idx_voice_restrictions_space_id", + "columns": [ + "space_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "voice_restrictions_space_id_spaces_id_fk": { + "name": "voice_restrictions_space_id_spaces_id_fk", + "tableFrom": "voice_restrictions", + "tableTo": "spaces", + "columnsFrom": [ + "space_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "voice_restrictions_user_id_users_id_fk": { + "name": "voice_restrictions_user_id_users_id_fk", + "tableFrom": "voice_restrictions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "voice_restrictions_moderator_id_users_id_fk": { + "name": "voice_restrictions_moderator_id_users_id_fk", + "tableFrom": "voice_restrictions", + "tableTo": "users", + "columnsFrom": [ + "moderator_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "voice_restrictions_space_id_user_id_restriction_type_pk": { + "columns": [ + "space_id", + "user_id", + "restriction_type" + ], + "name": "voice_restrictions_space_id_user_id_restriction_type_pk" + } + }, + "uniqueConstraints": {} + } + }, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index 293e21fd..251eabe7 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -57,6 +57,13 @@ "when": 1782832912087, "tag": "0007_nervous_orphan", "breakpoints": true + }, + { + "idx": 8, + "version": "6", + "when": 1782932926154, + "tag": "0008_cute_sebastian_shaw", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/server/src/db/migrate.test.ts b/packages/server/src/db/migrate.test.ts new file mode 100644 index 00000000..bfd6792c --- /dev/null +++ b/packages/server/src/db/migrate.test.ts @@ -0,0 +1,30 @@ +import { describe, it, expect } from 'vitest'; +import Database from 'better-sqlite3'; +import { ensureDefaults } from './migrate.js'; + +function freshDb(): Database.Database { + const db = new Database(':memory:'); + db.exec(`CREATE TABLE instance_settings (id integer PRIMARY KEY, worker_id integer, instance_id text, max_bitrate_kbps integer, min_bitrate_kbps integer, bitrate_step_kbps integer, allowed_resolutions text, allowed_framerates text, max_resolution integer, max_framerate integer, updated_at integer); + CREATE TABLE users (id text PRIMARY KEY, is_admin integer DEFAULT 0, created_at integer);`); + return db; +} + +describe('ensureDefaults instance epoch', () => { + it('mints an instance_id when null and is idempotent', () => { + const db = freshDb(); + ensureDefaults(db); + const first = (db.prepare('SELECT instance_id FROM instance_settings WHERE id = 1').get() as { instance_id: string }).instance_id; + expect(first).toMatch(/^[0-9a-f-]{36}$/); + ensureDefaults(db); + const second = (db.prepare('SELECT instance_id FROM instance_settings WHERE id = 1').get() as { instance_id: string }).instance_id; + expect(second).toBe(first); // stable across boots + }); + + it('mints a different id for a separate fresh DB', () => { + const a = freshDb(); ensureDefaults(a); + const b = freshDb(); ensureDefaults(b); + const idA = (a.prepare('SELECT instance_id FROM instance_settings WHERE id = 1').get() as { instance_id: string }).instance_id; + const idB = (b.prepare('SELECT instance_id FROM instance_settings WHERE id = 1').get() as { instance_id: string }).instance_id; + expect(idA).not.toBe(idB); + }); +}); diff --git a/packages/server/src/db/migrate.ts b/packages/server/src/db/migrate.ts index 70448c64..7375d409 100644 --- a/packages/server/src/db/migrate.ts +++ b/packages/server/src/db/migrate.ts @@ -27,6 +27,18 @@ export function ensureDefaults(db: Database.Database): void { console.log(`[defaults] Generated Snowflake worker ID: ${workerId}`); } + // 2b. Ensure a persistent instance epoch (incarnation UUID) exists. A fresh + // DB mints a new one — this is the discriminator for detecting resets. + // The id=1 row is guaranteed by step 1's INSERT OR IGNORE above. + const epochRow = db.prepare('SELECT instance_id FROM instance_settings WHERE id = 1').get() as + { instance_id: string | null } | undefined; + if (!epochRow || epochRow.instance_id === null) { + const instanceId = crypto.randomUUID(); + const res = db.prepare('UPDATE instance_settings SET instance_id = ? WHERE id = 1').run(instanceId); + if (res.changes !== 1) throw new Error('ensureDefaults: instance_settings id=1 row missing — cannot mint epoch'); + console.log('[defaults] Generated instance epoch'); + } + // 3. Ensure at least one admin exists (promote earliest registered user) const anyAdmin = db.prepare('SELECT id FROM users WHERE is_admin = 1 LIMIT 1').get(); if (!anyAdmin) { diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index 4d83df00..78d95390 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -23,6 +23,8 @@ export const users = sqliteTable('users', { passwordChangedAt: integer('password_changed_at'), showActivity: integer('show_activity').notNull().default(1), federationRegistryUpdatedAt: integer('federation_registry_updated_at').default(0), + federationHealPending: integer('federation_heal_pending').default(0), + federationHomeOrphaned: integer('federation_home_orphaned').default(0), createdAt: integer('created_at').notNull(), }); @@ -308,6 +310,7 @@ export const instanceSettings = sqliteTable('instance_settings', { id: integer('id').primaryKey().default(1), instanceName: text('instance_name').default('Backspace'), workerId: integer('worker_id'), + instanceId: text('instance_id'), discoveryEnabled: integer('discovery_enabled').notNull().default(1), maxBitrateKbps: integer('max_bitrate_kbps').notNull().default(20000), minBitrateKbps: integer('min_bitrate_kbps').notNull().default(500), @@ -385,6 +388,22 @@ export const federationPeers = sqliteTable('federation_peers', { autoRotateIntervalDays: integer('auto_rotate_interval_days').notNull().default(90), createdAt: integer('created_at').notNull(), approvalToken: text('approval_token'), + peerInstanceId: text('peer_instance_id'), + observedPeerInstanceId: text('observed_peer_instance_id'), + needsAttentionReason: text('needs_attention_reason'), +}); + +// Records a detected federated-peer reset (same origin, new instance epoch). +// One row per origin; upserted when a live epoch change is observed, resolved +// once stale replicated identities are healed. +export const federationResetEvents = sqliteTable('federation_reset_events', { + origin: text('origin').primaryKey(), + deadEpoch: text('dead_epoch').notNull(), + newEpoch: text('new_epoch'), + detectedAt: integer('detected_at').notNull(), + resolvedAt: integer('resolved_at'), + stubCount: integer('stub_count').notNull().default(0), + orphanedAccountCount: integer('orphaned_account_count').notNull().default(0), }); // SQL-level CHECK constraint enforces (direction='inbound' → hmac_secret NOT NULL). From 7acf48d0a49d3ab8872c64371e3fbc0e068500a2 Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Wed, 1 Jul 2026 21:19:25 +0200 Subject: [PATCH 02/12] feat(federation): shared epoch types + getInstanceId() --- docs/systems/admin.md | 3 + packages/server/src/routes/instance.test.ts | 7 +- packages/server/src/routes/instance.ts | 2 + .../server/src/utils/federationEpoch.test.ts | 84 +++++++++++++++++++ packages/server/src/utils/federationEpoch.ts | 24 ++++++ packages/shared/src/types.ts | 12 +++ 6 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/utils/federationEpoch.test.ts create mode 100644 packages/server/src/utils/federationEpoch.ts diff --git a/docs/systems/admin.md b/docs/systems/admin.md index 97e807f7..5bdf0e50 100644 --- a/docs/systems/admin.md +++ b/docs/systems/admin.md @@ -171,9 +171,12 @@ No authentication. Returns: federatedRegistrationOpen: boolean; // NOT NULL DEFAULT 1; gates federated-account creation sourceCodeUrl: string; // AGPL § 13; config.sourceCodeUrl (env BACKSPACE_SOURCE_URL) commit: string | null; // AGPL § 13; config.commit (env BACKSPACE_COMMIT, build-injected) + instanceId: string; // Persistent per-instance epoch (incarnation UUID); getInstanceId() } ``` +`instanceId` is the persistent per-instance epoch — a UUID minted once by `ensureDefaults` on first boot and stable across restarts (stored in `instance_settings.instance_id`, guaranteed non-null after boot). It changes only when the instance is wiped/re-provisioned. Peers read it to detect that a remote has been re-provisioned (federation epoch self-healing). The server reads it via the cached `getInstanceId()` in `utils/federationEpoch.ts`, which throws if the epoch is unset (invariant: `ensureDefaults` runs before any read). + Registration resolution order: `instance_settings.registrationOpen` (if not null) > `config.registrationOpen` (from `REGISTRATION_OPEN` env, default true). `federatedRegistrationOpen` is consumed by the Connections UI (client-federation.md) to decide whether to surface the "create federated account on this instance" affordance. diff --git a/packages/server/src/routes/instance.test.ts b/packages/server/src/routes/instance.test.ts index 3d0b8cc6..2eb2c808 100644 --- a/packages/server/src/routes/instance.test.ts +++ b/packages/server/src/routes/instance.test.ts @@ -53,9 +53,11 @@ beforeEach(async () => { // Seed the singleton instance_settings row mirroring ensureDefaults() — // tests don't run the boot-time helper, so we insert manually with the - // schema-default values for the new federatedRegistrationOpen column. + // schema-default values for the new federatedRegistrationOpen column plus + // the persistent epoch (instanceId) that ensureDefaults mints on boot. testDb.insert(schema.instanceSettings).values({ id: 1, + instanceId: '123e4567-e89b-12d3-a456-426614174000', updatedAt: Date.now(), }).run(); @@ -94,5 +96,8 @@ describe('GET /api/instance/info', () => { expect(typeof body.sourceCodeUrl).toBe('string'); expect(body.sourceCodeUrl).toMatch(/^https?:\/\//); expect(body.commit === null || typeof body.commit === 'string').toBe(true); + // Persistent per-instance epoch (incarnation UUID) is always advertised. + expect(typeof body.instanceId).toBe('string'); + expect(body.instanceId).toMatch(/^[0-9a-f-]{36}$/); }); }); diff --git a/packages/server/src/routes/instance.ts b/packages/server/src/routes/instance.ts index 730d6c13..5214c04e 100644 --- a/packages/server/src/routes/instance.ts +++ b/packages/server/src/routes/instance.ts @@ -2,6 +2,7 @@ import type { FastifyInstance } from 'fastify'; import { eq } from 'drizzle-orm'; import { getDb, schema } from '../db/index.js'; import { config } from '../config.js'; +import { getInstanceId } from '../utils/federationEpoch.js'; import type { InstanceInfoResponse } from '@backspace/shared'; const BACKSPACE_VERSION = '1.0.0'; @@ -23,6 +24,7 @@ export async function instanceRoutes(app: FastifyInstance): Promise { version: BACKSPACE_VERSION, registrationOpen, federatedRegistrationOpen: settings?.federatedRegistrationOpen === 1, + instanceId: getInstanceId(), // AGPL-3.0 § 13: advertise the source of the running version to every // network user (and federated peer) — public/unauthenticated by design. sourceCodeUrl: config.sourceCodeUrl, diff --git a/packages/server/src/utils/federationEpoch.test.ts b/packages/server/src/utils/federationEpoch.test.ts new file mode 100644 index 00000000..733a0187 --- /dev/null +++ b/packages/server/src/utils/federationEpoch.test.ts @@ -0,0 +1,84 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import Database from 'better-sqlite3'; +import { drizzle } from 'drizzle-orm/better-sqlite3'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import * as schema from '../db/schema.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +let sqlite: Database.Database; +let testDb: ReturnType>; + +vi.mock('../db/index.js', () => ({ + getDb: () => testDb, + getRawDb: () => sqlite, + schema, +})); + +function applyMigrations(db: Database.Database): void { + const dir = path.resolve(__dirname, '../../drizzle'); + for (const f of fs.readdirSync(dir).filter(f => f.endsWith('.sql')).sort()) { + const sqlText = fs.readFileSync(path.join(dir, f), 'utf8'); + for (const stmt of sqlText.split(/-->\s*statement-breakpoint/)) { + const clean = stmt.trim(); + if (clean) db.exec(clean); + } + } +} + +function seedEpoch(instanceId: string): void { + testDb.insert(schema.instanceSettings).values({ + id: 1, + instanceId, + updatedAt: Date.now(), + } as typeof schema.instanceSettings.$inferInsert).run(); +} + +beforeEach(async () => { + sqlite = new Database(':memory:'); + testDb = drizzle(sqlite, { schema }); + applyMigrations(sqlite); + const { __resetInstanceIdCacheForTest } = await import('./federationEpoch.js'); + __resetInstanceIdCacheForTest(); +}); + +describe('getInstanceId', () => { + it('returns the persisted epoch', async () => { + seedEpoch('123e4567-e89b-12d3-a456-426614174000'); + const { getInstanceId } = await import('./federationEpoch.js'); + const id = getInstanceId(); + expect(id).toBe('123e4567-e89b-12d3-a456-426614174000'); + expect(id).toMatch(/^[0-9a-f-]{36}$/); + }); + + it('caches the value after the first read', async () => { + seedEpoch('123e4567-e89b-12d3-a456-426614174000'); + const { getInstanceId } = await import('./federationEpoch.js'); + expect(getInstanceId()).toBe('123e4567-e89b-12d3-a456-426614174000'); + + // Mutate the underlying row; a cached reader must NOT observe the change. + testDb.update(schema.instanceSettings) + .set({ instanceId: 'ffffffff-ffff-ffff-ffff-ffffffffffff' }) + .run(); + expect(getInstanceId()).toBe('123e4567-e89b-12d3-a456-426614174000'); + }); + + it('re-reads after __resetInstanceIdCacheForTest clears the cache', async () => { + seedEpoch('123e4567-e89b-12d3-a456-426614174000'); + const { getInstanceId, __resetInstanceIdCacheForTest } = await import('./federationEpoch.js'); + expect(getInstanceId()).toBe('123e4567-e89b-12d3-a456-426614174000'); + + testDb.update(schema.instanceSettings) + .set({ instanceId: 'ffffffff-ffff-ffff-ffff-ffffffffffff' }) + .run(); + __resetInstanceIdCacheForTest(); + expect(getInstanceId()).toBe('ffffffff-ffff-ffff-ffff-ffffffffffff'); + }); + + it('throws when the epoch is unset (invariant: ensureDefaults must run first)', async () => { + // No row seeded — instance_settings is empty. + const { getInstanceId } = await import('./federationEpoch.js'); + expect(() => getInstanceId()).toThrow(/instance_id is not set/); + }); +}); diff --git a/packages/server/src/utils/federationEpoch.ts b/packages/server/src/utils/federationEpoch.ts new file mode 100644 index 00000000..daa18834 --- /dev/null +++ b/packages/server/src/utils/federationEpoch.ts @@ -0,0 +1,24 @@ +import { eq } from 'drizzle-orm'; +import { getDb, schema } from '../db/index.js'; + +let cached: string | null = null; + +/** This instance's persistent epoch (incarnation UUID). Set by ensureDefaults on boot. */ +export function getInstanceId(): string { + if (cached) return cached; + const db = getDb(); + const row = db.select({ instanceId: schema.instanceSettings.instanceId }) + .from(schema.instanceSettings) + .where(eq(schema.instanceSettings.id, 1)) + .get(); + if (!row?.instanceId) { + throw new Error('instance_id is not set — ensureDefaults must run before getInstanceId'); + } + cached = row.instanceId; + return cached; +} + +/** Test-only: clear the module cache between cases. */ +export function __resetInstanceIdCacheForTest(): void { + cached = null; +} diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 4e69377f..aa2796a7 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -798,6 +798,10 @@ export interface InstanceInfoResponse { version: string; registrationOpen: boolean; federatedRegistrationOpen: boolean; + // Persistent per-instance epoch (incarnation UUID). Minted by ensureDefaults on + // first boot and stable across restarts; changes only on a wipe/re-provision. + // Peers use it to detect that a remote has been re-provisioned (self-healing). + instanceId: string; // AGPL-3.0 § 13 network-use source offer: URL to the Corresponding Source of // the version this instance is running (operator-configurable via // BACKSPACE_SOURCE_URL so forks point at their own source). @@ -1111,9 +1115,17 @@ export interface FederationRelayAttachment { export interface FederationRelayRequest { version: 1; sourceInstance: string; + // Sender's persistent epoch (incarnation UUID). Optional for wire compatibility + // with peers that predate epoch self-healing; when present, the receiver can + // detect that the source instance has been re-provisioned. + sourceInstanceId?: string; events: FederationRelayEvent[]; } +export interface FederationEpochResponse { + instanceId: string; +} + export interface FederationRelayResponse { accepted: string[]; rejected: Array<{ messageId: string; reason: string }>; From 538519fcd2d5ca570d50bcaca79f7cc690230742 Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Wed, 1 Jul 2026 21:35:20 +0200 Subject: [PATCH 03/12] feat(federation): exchange + store peer epoch on handshake --- docs/systems/federation.md | 14 +- .../routes/federation.approveOutbound.test.ts | 1 + .../routes/federation.outboundApprove.test.ts | 1 + .../routes/federation.outboundDeny.test.ts | 1 + .../src/routes/federation.peerAccept.test.ts | 1 + ...federation.peerAcceptApprovalToken.test.ts | 1 + .../federation.peerInitiateOutbound.test.ts | 1 + packages/server/src/routes/federation.ts | 64 ++-- .../utils/federationEpochHandshake.test.ts | 301 ++++++++++++++++++ .../federationPeering.approvalToken.test.ts | 1 + .../federationPeering.instanceName.test.ts | 1 + .../federationPeering.outboundGate.test.ts | 1 + .../server/src/utils/federationPeering.ts | 17 +- 13 files changed, 375 insertions(+), 30 deletions(-) create mode 100644 packages/server/src/utils/federationEpochHandshake.test.ts diff --git a/docs/systems/federation.md b/docs/systems/federation.md index d7d1127a..ec9121d2 100644 --- a/docs/systems/federation.md +++ b/docs/systems/federation.md @@ -57,17 +57,19 @@ Backspace federation is peer-to-peer with no central authority. Each instance ma - Validates `sourceOrigin`, `challenge`, `hmacSecret`, and (optional) `instanceName` from body - Handles existing peers: active -> return 200 (idempotent), revoked -> return 403, pending -> update with new secret and activate - New peer: creates record with provided `hmacSecret`, sets `status='active'` -- Returns `{ accepted: true, instanceName: }` on success — see "Instance name exchange" below +- Returns `{ accepted: true, instanceName: , instanceId: }` on success — see "Instance name & epoch exchange" below -### Instance name exchange +### Instance name & epoch exchange -The handshake is bidirectional for the `instance_name` label rendered in the federation panel and in DM-call toasts (`peerLabel`): +The handshake is bidirectional for two pieces of metadata: the `instance_name` label rendered in the federation panel and in DM-call toasts (`peerLabel`), and the **instance epoch** (`instance_id`, this instance's persistent incarnation UUID minted by `ensureDefaults`, accessed via `getInstanceId()`). The epoch is the authenticated baseline used by the instance-epoch self-healing feature to detect a wipe-and-reinstall on the same domain (design: `docs/superpowers/specs/2026-07-01-federation-instance-epoch-self-healing-design.md`). -- **Initiator → responder:** the request body to `/peer/accept` carries `{ sourceOrigin, hmacSecret, instanceName }`. The responder reads `instanceName` and persists it to `federation_peers.instance_name` on every state-mutating activation path: `pending → active`, `awaiting_approval → active`, `rejected → active` (override), and new-peer create. The idempotent early-return path for already-`active` and `needs_attention` peers does NOT overwrite — same security posture that already refuses to overwrite `hmac_secret` on these paths from an unauthenticated request. +- **Initiator → responder:** the request body to `/peer/accept` carries `{ sourceOrigin, hmacSecret, instanceName, instanceId }`. The responder reads `instanceName` → `federation_peers.instance_name` and `instanceId` → `federation_peers.peer_instance_id` on every state-mutating activation path: `pending → active`, `awaiting_approval → active` (token-valid and autoAccept-fallback), `rejected → active` (override), and new-peer create. The idempotent early-return path for already-`active` and `needs_attention` peers does NOT overwrite — same security posture that already refuses to overwrite `hmac_secret` on these paths from an unauthenticated request. (The idempotent guard's *detection* of a changed epoch on that path is a later part of the self-healing feature; the handshake itself only writes the epoch on true activation.) -- **Responder → initiator:** the `/peer/accept` response body is `{ accepted: true, instanceName: }`. The initiator (`performHandshake` in `utils/federationPeering.ts` and `/peer/initiate` in `routes/federation.ts`) parses it and persists alongside the `status='active'` write. Older peers that omit the field are tolerated — the column stays `null`. Non-JSON bodies are tolerated defensively. +- **Responder → initiator:** the `/peer/accept` response body is `{ accepted: true, instanceName: , instanceId: }`. The initiator (`performHandshake` in `utils/federationPeering.ts`, `/peer/initiate`, and both `/approval-requests/:id/approve` handlers in `routes/federation.ts`) parses `instanceName` and `instanceId` and persists them alongside the `status='active'` write (`peer_instance_id`). Older peers that omit either field are tolerated — the respective column stays `null` (backstopped later by the deterministic epoch-refresh and relay-envelope population). Non-JSON bodies are tolerated defensively. -`instance_name` is cosmetic metadata, eventually-consistent. Anywhere `peerLabel` is rendered falls back to origin hostname when `instance_name IS NULL`. Instance renames do not currently re-broadcast — that's a separate, unimplemented feature. +All four outbound `/peer/accept` senders (`performHandshake`, `/peer/initiate`, and the inbound + outbound `/approve` handlers) include `instanceId: getInstanceId()` in the request body, so a peer learns our epoch regardless of which path activated the relationship. + +`instance_name` is cosmetic metadata, eventually-consistent. Anywhere `peerLabel` is rendered falls back to origin hostname when `instance_name IS NULL`. Instance renames do not currently re-broadcast — that's a separate, unimplemented feature. `peer_instance_id` is trust-consequential (only ever written from authenticated channels) — see the self-healing design spec for detection/heal semantics. ### Secret Storage & Rotation diff --git a/packages/server/src/routes/federation.approveOutbound.test.ts b/packages/server/src/routes/federation.approveOutbound.test.ts index 5683fb57..28e9c46e 100644 --- a/packages/server/src/routes/federation.approveOutbound.test.ts +++ b/packages/server/src/routes/federation.approveOutbound.test.ts @@ -81,6 +81,7 @@ function seedInstanceSettings(): void { testDb.insert(schema.instanceSettings).values({ id: 1, instanceName: 'Local Backspace', + instanceId: 'test-epoch-local', autoAcceptPeering: 0, registrationOpen: 1, updatedAt: Date.now(), diff --git a/packages/server/src/routes/federation.outboundApprove.test.ts b/packages/server/src/routes/federation.outboundApprove.test.ts index 80e38b63..868e8434 100644 --- a/packages/server/src/routes/federation.outboundApprove.test.ts +++ b/packages/server/src/routes/federation.outboundApprove.test.ts @@ -90,6 +90,7 @@ function seedInstanceSettings(): void { testDb.insert(schema.instanceSettings).values({ id: 1, instanceName: 'Local Backspace', + instanceId: 'test-epoch-local', autoAcceptPeering: 0, registrationOpen: 1, updatedAt: Date.now(), diff --git a/packages/server/src/routes/federation.outboundDeny.test.ts b/packages/server/src/routes/federation.outboundDeny.test.ts index 27f66bc1..6d566f7c 100644 --- a/packages/server/src/routes/federation.outboundDeny.test.ts +++ b/packages/server/src/routes/federation.outboundDeny.test.ts @@ -85,6 +85,7 @@ function seedInstanceSettings(): void { testDb.insert(schema.instanceSettings).values({ id: 1, instanceName: 'Local Backspace', + instanceId: 'test-epoch-local', autoAcceptPeering: 0, registrationOpen: 1, updatedAt: Date.now(), diff --git a/packages/server/src/routes/federation.peerAccept.test.ts b/packages/server/src/routes/federation.peerAccept.test.ts index 6ce1e1c3..0e9bad04 100644 --- a/packages/server/src/routes/federation.peerAccept.test.ts +++ b/packages/server/src/routes/federation.peerAccept.test.ts @@ -62,6 +62,7 @@ function seedInstanceSettings(name: string): void { testDb.insert(schema.instanceSettings).values({ id: 1, instanceName: name, + instanceId: 'test-epoch-local', autoAcceptPeering: 1, registrationOpen: 1, updatedAt: Date.now(), diff --git a/packages/server/src/routes/federation.peerAcceptApprovalToken.test.ts b/packages/server/src/routes/federation.peerAcceptApprovalToken.test.ts index 933741f4..e2a4262c 100644 --- a/packages/server/src/routes/federation.peerAcceptApprovalToken.test.ts +++ b/packages/server/src/routes/federation.peerAcceptApprovalToken.test.ts @@ -61,6 +61,7 @@ function seedInstanceSettings(autoAccept: 0 | 1): void { testDb.insert(schema.instanceSettings).values({ id: 1, instanceName: 'Local Backspace', + instanceId: 'test-epoch-local', autoAcceptPeering: autoAccept, registrationOpen: 1, updatedAt: Date.now(), diff --git a/packages/server/src/routes/federation.peerInitiateOutbound.test.ts b/packages/server/src/routes/federation.peerInitiateOutbound.test.ts index daf5e9a7..328caa23 100644 --- a/packages/server/src/routes/federation.peerInitiateOutbound.test.ts +++ b/packages/server/src/routes/federation.peerInitiateOutbound.test.ts @@ -81,6 +81,7 @@ function seedInstanceSettings(): void { testDb.insert(schema.instanceSettings).values({ id: 1, instanceName: 'Local Backspace', + instanceId: 'test-epoch-local', autoAcceptPeering: 0, registrationOpen: 1, updatedAt: Date.now(), diff --git a/packages/server/src/routes/federation.ts b/packages/server/src/routes/federation.ts index 8f912f1c..378537a1 100644 --- a/packages/server/src/routes/federation.ts +++ b/packages/server/src/routes/federation.ts @@ -19,6 +19,7 @@ import { deleteAttachmentFiles, deleteUploadFile } from '../utils/fileCleanup.js import { tombstoneUser, collectDeletionBroadcastTargets, collectProfileBroadcastTargetIds } from '../utils/userDeletion.js'; import { computeFederatedId, getDmParticipants, sendCallRelay } from '../utils/federationOutbox.js'; import { onPeerActivated, onPeerDeactivated } from '../utils/federationPeerActivation.js'; +import { getInstanceId } from '../utils/federationEpoch.js'; import { probePeerReachable, markPeerRecovered } from '../utils/federationRecovery.js'; import { getDmMessageWithUser } from './dm.js'; import type { FederationRelayRequest, FederationRelayResponse, FederationRelayEvent, FederationRelayAttachment, FederationSyncRequest, FederationSyncResponse, DmMessageWithUser, DmChannel, FederationRelayProfileSnapshot, FederationIdentityDeleteS2SRequest, FederationProfileUpdatePayload, ServerEvent, ApprovalRequestSubscriberSummary, PeeringTriggerReason } from '@backspace/shared'; @@ -379,6 +380,7 @@ async function handleInboundApprove( sourceOrigin: localOrigin, hmacSecret, instanceName, + instanceId: getInstanceId(), // Forward the stored token (issued in our 202 response when the // remote first sent /peer/accept). Lets the remote verify mutual // admin approval. Spec §3.7. @@ -430,21 +432,26 @@ async function handleInboundApprove( return reply.code(502).send({ error: errorMessage, statusCode: 502 }); } - // Parse the remote's instanceName from the response body so the - // federation panel renders a friendly label. Tolerate omission and - // non-JSON bodies — same pattern as performHandshake and /peer/initiate. + // Parse the remote's instanceName and instanceId (epoch) from the response + // body so the federation panel renders a friendly label and we record the + // peer's authenticated epoch baseline. Tolerate omission and non-JSON + // bodies — same pattern as performHandshake and /peer/initiate. let remoteInstanceName: string | null = null; + let remoteInstanceId: string | null = null; try { - const body = (await response.json()) as { instanceName?: string | null }; + const body = (await response.json()) as { instanceName?: string | null; instanceId?: string | null }; if (typeof body?.instanceName === 'string' && body.instanceName.length > 0) { remoteInstanceName = body.instanceName; } + if (typeof body?.instanceId === 'string' && body.instanceId.length > 0) { + remoteInstanceId = body.instanceId; + } } catch { // Non-JSON body — leave null. } db.update(schema.federationPeers) - .set({ status: 'active', lastSeenAt: now, instanceName: remoteInstanceName, approvalToken: null }) + .set({ status: 'active', lastSeenAt: now, instanceName: remoteInstanceName, peerInstanceId: remoteInstanceId, approvalToken: null }) .where(eq(schema.federationPeers.id, peerId)) .run(); @@ -532,6 +539,7 @@ async function handleOutboundApprove( sourceOrigin: localOrigin, hmacSecret, instanceName, + instanceId: getInstanceId(), // No approvalToken — outbound rows are admin-initiated locally; we // hold no prior token from the remote and rely on the remote's own // autoAcceptPeering setting to decide 200 vs 202. @@ -612,13 +620,18 @@ async function handleOutboundApprove( }); } - // 200 — peer activated. Capture remote's instanceName for the friendly label. + // 200 — peer activated. Capture remote's instanceName for the friendly label + // and instanceId (epoch) for the authenticated baseline. let remoteInstanceName: string | null = approvalReq.instanceName; + let remoteInstanceId: string | null = null; try { - const body = (await response.json()) as { instanceName?: string | null }; + const body = (await response.json()) as { instanceName?: string | null; instanceId?: string | null }; if (typeof body?.instanceName === 'string' && body.instanceName.length > 0) { remoteInstanceName = body.instanceName; } + if (typeof body?.instanceId === 'string' && body.instanceId.length > 0) { + remoteInstanceId = body.instanceId; + } } catch { // Non-JSON body — keep approvalReq.instanceName (may be null). } @@ -628,6 +641,7 @@ async function handleOutboundApprove( status: 'active', lastSeenAt: now, instanceName: remoteInstanceName, + peerInstanceId: remoteInstanceId, approvalToken: null, }) .where(eq(schema.federationPeers.id, peerId)) @@ -883,6 +897,7 @@ export async function federationRoutes(app: FastifyInstance): Promise { .from(schema.instanceSettings) .where(eq(schema.instanceSettings.id, 1)) .get()?.name ?? undefined, + instanceId: getInstanceId(), }), signal: AbortSignal.timeout(10_000), }); @@ -940,20 +955,25 @@ export async function federationRoutes(app: FastifyInstance): Promise { } // Remote accepted — activate the peer. Parse the remote's instanceName - // from the response body so the federation panel renders a friendly - // label. Tolerate omission and non-JSON bodies. + // and instanceId (epoch) from the response body so the federation panel + // renders a friendly label and we record the peer's authenticated + // epoch baseline. Tolerate omission and non-JSON bodies. let remoteInstanceName: string | null = null; + let remoteInstanceId: string | null = null; try { - const body = (await response.json()) as { instanceName?: string | null }; + const body = (await response.json()) as { instanceName?: string | null; instanceId?: string | null }; if (typeof body?.instanceName === 'string' && body.instanceName.length > 0) { remoteInstanceName = body.instanceName; } + if (typeof body?.instanceId === 'string' && body.instanceId.length > 0) { + remoteInstanceId = body.instanceId; + } } catch { // Non-JSON body — leave null. } db.update(schema.federationPeers) - .set({ status: 'active', lastSeenAt: Date.now(), instanceName: remoteInstanceName, approvalToken: null }) + .set({ status: 'active', lastSeenAt: Date.now(), instanceName: remoteInstanceName, peerInstanceId: remoteInstanceId, approvalToken: null }) .where(eq(schema.federationPeers.id, peerId)) .run(); connectionManager.sendToAdmins({ type: 'federation_peers_changed' as const }); @@ -994,7 +1014,7 @@ export async function federationRoutes(app: FastifyInstance): Promise { // ─── POST /api/federation/peer/accept ────────────────────────────────────── // Server-to-server: accept a peering request from a remote instance. // No JWT auth — this is first contact. Rate-limited by IP. - app.post<{ Body: { sourceOrigin: string; challenge?: string; hmacSecret: string; instanceName?: string; approvalToken?: string } }>( + app.post<{ Body: { sourceOrigin: string; challenge?: string; hmacSecret: string; instanceName?: string; instanceId?: string; approvalToken?: string } }>( '/api/federation/peer/accept', async (request, reply) => { const clientIp = request.ip; @@ -1005,7 +1025,7 @@ export async function federationRoutes(app: FastifyInstance): Promise { }); } - const { sourceOrigin: rawOrigin, hmacSecret, instanceName: reqInstanceName, approvalToken: inboundToken } = request.body ?? {}; + const { sourceOrigin: rawOrigin, hmacSecret, instanceName: reqInstanceName, instanceId: reqInstanceId, approvalToken: inboundToken } = request.body ?? {}; if (!rawOrigin || typeof rawOrigin !== 'string') { return reply.code(400).send({ error: 'sourceOrigin is required', statusCode: 400 }); @@ -1031,6 +1051,7 @@ export async function federationRoutes(app: FastifyInstance): Promise { .get(); const ourInstanceName = settings?.instanceName ?? null; + const ourInstanceId = getInstanceId(); const autoAccept = settings?.autoAcceptPeering ?? 1; // ── autoAcceptPeering gate ────────────────────────────────────────── @@ -1099,7 +1120,7 @@ export async function federationRoutes(app: FastifyInstance): Promise { // Legitimate recovery path: local admin clicks "Reset peering" → // row is deleted → remote's /peer/accept then lands on a // non-existent row and the normal handshake path runs. - return reply.code(200).send({ accepted: true, instanceName: ourInstanceName }); + return reply.code(200).send({ accepted: true, instanceName: ourInstanceName, instanceId: ourInstanceId }); } if (existing.status === 'revoked') { return reply.code(403).send({ @@ -1114,6 +1135,7 @@ export async function federationRoutes(app: FastifyInstance): Promise { .set({ hmacSecret, instanceName: reqInstanceName ?? null, + peerInstanceId: reqInstanceId ?? null, status: 'active', lastSeenAt: Date.now(), }) @@ -1133,7 +1155,7 @@ export async function federationRoutes(app: FastifyInstance): Promise { console.error('[federation] onPeerActivated from /peer/accept (rejected override) failed:', err) ); - return reply.code(200).send({ accepted: true, instanceName: ourInstanceName }); + return reply.code(200).send({ accepted: true, instanceName: ourInstanceName, instanceId: ourInstanceId }); } if (existing.status === 'awaiting_approval') { // Spec §3.5: token verification gates the awaiting_approval → active @@ -1153,6 +1175,7 @@ export async function federationRoutes(app: FastifyInstance): Promise { .set({ hmacSecret, instanceName: reqInstanceName ?? null, + peerInstanceId: reqInstanceId ?? null, status: 'active', lastSeenAt: Date.now(), approvalToken: null, @@ -1176,7 +1199,7 @@ export async function federationRoutes(app: FastifyInstance): Promise { onPeerActivated(existing.id, 'accept_awaiting_approval').catch(err => console.error('[federation] onPeerActivated from /peer/accept (awaiting_approval) failed:', err) ); - return reply.code(200).send({ accepted: true, instanceName: ourInstanceName }); + return reply.code(200).send({ accepted: true, instanceName: ourInstanceName, instanceId: ourInstanceId }); } // Token absent or mismatched. Cannot prove mutual approval. @@ -1188,6 +1211,7 @@ export async function federationRoutes(app: FastifyInstance): Promise { .set({ hmacSecret, instanceName: reqInstanceName ?? null, + peerInstanceId: reqInstanceId ?? null, status: 'active', lastSeenAt: Date.now(), approvalToken: null, @@ -1205,7 +1229,7 @@ export async function federationRoutes(app: FastifyInstance): Promise { onPeerActivated(existing.id, 'accept_awaiting_approval_fallback').catch(err => console.error('[federation] onPeerActivated from /peer/accept (awaiting_approval fallback) failed:', err) ); - return reply.code(200).send({ accepted: true, instanceName: ourInstanceName }); + return reply.code(200).send({ accepted: true, instanceName: ourInstanceName, instanceId: ourInstanceId }); } // autoAccept=0 + unverifiable inbound → queue as new approval-request. @@ -1218,6 +1242,7 @@ export async function federationRoutes(app: FastifyInstance): Promise { .set({ hmacSecret, instanceName: reqInstanceName ?? null, + peerInstanceId: reqInstanceId ?? null, status: 'active', lastSeenAt: Date.now(), }) @@ -1229,7 +1254,7 @@ export async function federationRoutes(app: FastifyInstance): Promise { console.error('[federation] onPeerActivated from /peer/accept (pending) failed:', err) ); - return reply.code(200).send({ accepted: true, instanceName: ourInstanceName }); + return reply.code(200).send({ accepted: true, instanceName: ourInstanceName, instanceId: ourInstanceId }); } // New peer — create and activate @@ -1239,6 +1264,7 @@ export async function federationRoutes(app: FastifyInstance): Promise { origin: sourceOrigin, hmacSecret, instanceName: reqInstanceName ?? null, + peerInstanceId: reqInstanceId ?? null, status: 'active', lastSeenAt: Date.now(), createdAt: Date.now(), @@ -1249,7 +1275,7 @@ export async function federationRoutes(app: FastifyInstance): Promise { console.error('[federation] onPeerActivated from /peer/accept (new) failed:', err) ); - return reply.code(200).send({ accepted: true, instanceName: ourInstanceName }); + return reply.code(200).send({ accepted: true, instanceName: ourInstanceName, instanceId: ourInstanceId }); }, ); diff --git a/packages/server/src/utils/federationEpochHandshake.test.ts b/packages/server/src/utils/federationEpochHandshake.test.ts new file mode 100644 index 00000000..d51054af --- /dev/null +++ b/packages/server/src/utils/federationEpochHandshake.test.ts @@ -0,0 +1,301 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import Fastify, { type FastifyInstance } from 'fastify'; +import Database from 'better-sqlite3'; +import { drizzle } from 'drizzle-orm/better-sqlite3'; +import { eq } from 'drizzle-orm'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import * as schema from '../db/schema.js'; +import { setWorkerId } from './snowflake.js'; + +setWorkerId(1); + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +type TestDb = ReturnType>; +let sqlite: Database.Database; +let testDb: TestDb; + +vi.mock('../db/index.js', () => ({ + getDb: () => testDb, + getRawDb: () => sqlite, + schema, +})); + +vi.mock('../utils/auth.js', () => ({ + authenticate: async (req: { userId?: string }) => { + req.userId = 'admin-user'; + }, + requireAdmin: async () => { + // peer/accept is unauthenticated anyway + }, +})); + +vi.mock('../ws/handler.js', () => ({ + connectionManager: { + sendToAdmins: vi.fn(), + getAllOnlineUserIds: () => [], + sendToUser: vi.fn(), + sendToDmMembers: vi.fn(), + }, +})); + +vi.mock('../utils/federationPeerActivation.js', () => ({ + onPeerActivated: vi.fn(async () => undefined), + onPeerDeactivated: vi.fn(async () => undefined), +})); + +const LOCAL_EPOCH = 'local-epoch-0000'; + +function applyMigrations(db: Database.Database): void { + const migrationsDir = path.resolve(__dirname, '../../drizzle'); + const files = fs.readdirSync(migrationsDir).filter(f => f.endsWith('.sql')).sort(); + for (const f of files) { + const sqlText = fs.readFileSync(path.join(migrationsDir, f), 'utf8'); + const statements = sqlText.split(/-->\s*statement-breakpoint/); + for (const stmt of statements) { + const clean = stmt.trim(); + if (clean) db.exec(clean); + } + } +} + +function seedInstanceSettings(): void { + testDb.insert(schema.instanceSettings).values({ + id: 1, + instanceName: 'Local Backspace', + instanceId: LOCAL_EPOCH, + autoAcceptPeering: 1, + registrationOpen: 1, + updatedAt: Date.now(), + }).run(); +} + +async function buildApp(): Promise { + const app = Fastify({ logger: false }); + const { federationRoutes } = await import('../routes/federation.js'); + await app.register(federationRoutes); + await app.ready(); + return app; +} + +describe('POST /api/federation/peer/accept — peer_instance_id (epoch) persistence', () => { + let app: FastifyInstance; + + beforeEach(async () => { + sqlite = new Database(':memory:'); + testDb = drizzle(sqlite, { schema }); + applyMigrations(sqlite); + seedInstanceSettings(); + const { __resetInstanceIdCacheForTest } = await import('./federationEpoch.js'); + __resetInstanceIdCacheForTest(); + app = await buildApp(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + sqlite.close(); + }); + + it('writes peer_instance_id when activating an existing pending peer', async () => { + testDb.insert(schema.federationPeers).values({ + id: 'peer-pending', + origin: 'https://remote.example', + hmacSecret: 'old-secret', + status: 'pending', + createdAt: Date.now(), + }).run(); + + const response = await app.inject({ + method: 'POST', + url: '/api/federation/peer/accept', + payload: { + sourceOrigin: 'https://remote.example', + hmacSecret: 'new-secret', + instanceName: 'Remote Backspace', + instanceId: 'epoch-A', + }, + }); + + expect(response.statusCode).toBe(200); + const row = testDb.select().from(schema.federationPeers) + .where(eq(schema.federationPeers.id, 'peer-pending')).get(); + expect(row?.status).toBe('active'); + expect(row?.peerInstanceId).toBe('epoch-A'); + }); + + it('writes peer_instance_id when creating a brand-new peer', async () => { + const response = await app.inject({ + method: 'POST', + url: '/api/federation/peer/accept', + payload: { + sourceOrigin: 'https://remote.example', + hmacSecret: 'remote-secret', + instanceName: 'Remote Backspace', + instanceId: 'epoch-B', + }, + }); + + expect(response.statusCode).toBe(200); + const row = testDb.select().from(schema.federationPeers) + .where(eq(schema.federationPeers.origin, 'https://remote.example')).get(); + expect(row?.status).toBe('active'); + expect(row?.peerInstanceId).toBe('epoch-B'); + }); + + it('writes peer_instance_id when overriding rejected → active', async () => { + testDb.insert(schema.federationPeers).values({ + id: 'peer-rejected', + origin: 'https://remote.example', + hmacSecret: 'old-secret', + status: 'rejected', + createdAt: Date.now(), + }).run(); + + const response = await app.inject({ + method: 'POST', + url: '/api/federation/peer/accept', + payload: { + sourceOrigin: 'https://remote.example', + hmacSecret: 'new-secret', + instanceId: 'epoch-C', + }, + }); + + expect(response.statusCode).toBe(200); + const row = testDb.select().from(schema.federationPeers) + .where(eq(schema.federationPeers.id, 'peer-rejected')).get(); + expect(row?.status).toBe('active'); + expect(row?.peerInstanceId).toBe('epoch-C'); + }); + + it('writes peer_instance_id on the awaiting_approval autoAccept fallback path', async () => { + testDb.insert(schema.federationPeers).values({ + id: 'peer-await', + origin: 'https://remote.example', + hmacSecret: 'old-secret', + status: 'awaiting_approval', + createdAt: Date.now(), + }).run(); + + const response = await app.inject({ + method: 'POST', + url: '/api/federation/peer/accept', + payload: { + sourceOrigin: 'https://remote.example', + hmacSecret: 'new-secret', + instanceId: 'epoch-D', + }, + }); + + expect(response.statusCode).toBe(200); + const row = testDb.select().from(schema.federationPeers) + .where(eq(schema.federationPeers.id, 'peer-await')).get(); + expect(row?.status).toBe('active'); + expect(row?.peerInstanceId).toBe('epoch-D'); + }); + + it('writes null peer_instance_id when body omits instanceId (legacy peer)', async () => { + const response = await app.inject({ + method: 'POST', + url: '/api/federation/peer/accept', + payload: { + sourceOrigin: 'https://remote.example', + hmacSecret: 'remote-secret', + instanceName: 'Remote Backspace', + }, + }); + + expect(response.statusCode).toBe(200); + const row = testDb.select().from(schema.federationPeers) + .where(eq(schema.federationPeers.origin, 'https://remote.example')).get(); + expect(row?.status).toBe('active'); + expect(row?.peerInstanceId).toBeNull(); + }); + + it('returns our own instanceId in the response body', async () => { + const response = await app.inject({ + method: 'POST', + url: '/api/federation/peer/accept', + payload: { + sourceOrigin: 'https://remote.example', + hmacSecret: 'remote-secret', + instanceId: 'epoch-E', + }, + }); + + expect(response.statusCode).toBe(200); + const body = response.json() as { accepted: boolean; instanceName?: string | null; instanceId?: string }; + expect(body.accepted).toBe(true); + expect(body.instanceId).toBe(LOCAL_EPOCH); + }); +}); + +describe('POST /api/federation/peer/initiate — persists remote epoch from handshake response', () => { + let app: FastifyInstance; + + beforeEach(async () => { + sqlite = new Database(':memory:'); + testDb = drizzle(sqlite, { schema }); + applyMigrations(sqlite); + seedInstanceSettings(); + const { __resetInstanceIdCacheForTest } = await import('./federationEpoch.js'); + __resetInstanceIdCacheForTest(); + app = await buildApp(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + sqlite.close(); + }); + + it('writes peer_instance_id from the remote /peer/accept response body', async () => { + const fetchMock = vi.fn(async () => + new Response(JSON.stringify({ accepted: true, instanceName: 'Remote', instanceId: 'remote-epoch-1' }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ); + vi.stubGlobal('fetch', fetchMock); + + const response = await app.inject({ + method: 'POST', + url: '/api/federation/peer/initiate', + payload: { remoteOrigin: 'https://remote.example' }, + }); + + expect(response.statusCode).toBe(200); + const row = testDb.select().from(schema.federationPeers) + .where(eq(schema.federationPeers.origin, 'https://remote.example')).get(); + expect(row?.status).toBe('active'); + expect(row?.peerInstanceId).toBe('remote-epoch-1'); + + // Our epoch must be sent in the outbound handshake body. + const call = fetchMock.mock.calls[0] as unknown as [string, RequestInit]; + const sentBody = JSON.parse(call[1].body as string) as { instanceId?: string }; + expect(sentBody.instanceId).toBe(LOCAL_EPOCH); + }); + + it('writes null peer_instance_id when the remote response omits instanceId', async () => { + vi.stubGlobal('fetch', vi.fn(async () => + new Response(JSON.stringify({ accepted: true, instanceName: 'Remote' }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + )); + + const response = await app.inject({ + method: 'POST', + url: '/api/federation/peer/initiate', + payload: { remoteOrigin: 'https://remote.example' }, + }); + + expect(response.statusCode).toBe(200); + const row = testDb.select().from(schema.federationPeers) + .where(eq(schema.federationPeers.origin, 'https://remote.example')).get(); + expect(row?.status).toBe('active'); + expect(row?.peerInstanceId).toBeNull(); + }); +}); diff --git a/packages/server/src/utils/federationPeering.approvalToken.test.ts b/packages/server/src/utils/federationPeering.approvalToken.test.ts index 385fe076..666a8150 100644 --- a/packages/server/src/utils/federationPeering.approvalToken.test.ts +++ b/packages/server/src/utils/federationPeering.approvalToken.test.ts @@ -70,6 +70,7 @@ function seedInstanceSettings(): void { testDb.insert(schema.instanceSettings).values({ id: 1, instanceName: 'Local Backspace', + instanceId: 'test-epoch-local', autoAcceptPeering: 1, registrationOpen: 1, updatedAt: Date.now(), diff --git a/packages/server/src/utils/federationPeering.instanceName.test.ts b/packages/server/src/utils/federationPeering.instanceName.test.ts index ca48bd33..46ac1e0c 100644 --- a/packages/server/src/utils/federationPeering.instanceName.test.ts +++ b/packages/server/src/utils/federationPeering.instanceName.test.ts @@ -69,6 +69,7 @@ function seedInstanceSettings(): void { testDb.insert(schema.instanceSettings).values({ id: 1, instanceName: 'Local Backspace', + instanceId: 'test-epoch-local', autoAcceptPeering: 1, registrationOpen: 1, updatedAt: Date.now(), diff --git a/packages/server/src/utils/federationPeering.outboundGate.test.ts b/packages/server/src/utils/federationPeering.outboundGate.test.ts index a189b190..7de0a61a 100644 --- a/packages/server/src/utils/federationPeering.outboundGate.test.ts +++ b/packages/server/src/utils/federationPeering.outboundGate.test.ts @@ -70,6 +70,7 @@ function seedInstanceSettings(autoAcceptPeering: 0 | 1): void { testDb.insert(schema.instanceSettings).values({ id: 1, instanceName: 'Local Backspace', + instanceId: 'test-epoch-local', autoAcceptPeering, registrationOpen: 1, updatedAt: Date.now(), diff --git a/packages/server/src/utils/federationPeering.ts b/packages/server/src/utils/federationPeering.ts index 4fdc8527..04da65cb 100644 --- a/packages/server/src/utils/federationPeering.ts +++ b/packages/server/src/utils/federationPeering.ts @@ -5,6 +5,7 @@ import { generateSnowflake } from './snowflake.js'; import { getOurOrigin, generateHmacSecret } from './federationAuth.js'; import { validateOrigin } from '../routes/federation.js'; import { onPeerActivated, onPeerDeactivated } from './federationPeerActivation.js'; +import { getInstanceId } from './federationEpoch.js'; import type { EnsurePeeredCallerIntent } from '@backspace/shared'; // ─── Types ─────────────────────────────────────────────────────────────────── @@ -304,6 +305,7 @@ async function performHandshake( sourceOrigin: ourOrigin, hmacSecret, instanceName: getInstanceName(), + instanceId: getInstanceId(), }), signal: AbortSignal.timeout(10_000), }); @@ -333,21 +335,26 @@ async function performHandshake( } if (response.ok) { - // 200 = peer accepted and activated. Parse remote's instanceName from - // the response body so we can render a friendly label for the peer. + // 200 = peer accepted and activated. Parse remote's instanceName and + // instanceId (epoch) from the response body so we can render a friendly + // label for the peer and record its authenticated epoch baseline. // Tolerate omission (older peers) and non-JSON bodies (defensive). let remoteInstanceName: string | null = null; + let remoteInstanceId: string | null = null; try { - const body = (await response.json()) as { instanceName?: string | null }; + const body = (await response.json()) as { instanceName?: string | null; instanceId?: string | null }; if (typeof body?.instanceName === 'string' && body.instanceName.length > 0) { remoteInstanceName = body.instanceName; } + if (typeof body?.instanceId === 'string' && body.instanceId.length > 0) { + remoteInstanceId = body.instanceId; + } } catch { - // Non-JSON or empty body — leave remoteInstanceName as null. + // Non-JSON or empty body — leave remoteInstanceName/Id as null. } db.update(schema.federationPeers) - .set({ status: 'active', lastSeenAt: Date.now(), instanceName: remoteInstanceName, approvalToken: null }) + .set({ status: 'active', lastSeenAt: Date.now(), instanceName: remoteInstanceName, peerInstanceId: remoteInstanceId, approvalToken: null }) .where(eq(schema.federationPeers.id, peerId)) .run(); const { connectionManager } = await import('../ws/handler.js'); From bf74aa8bb2d8c5e74a7588a4d02d262ec6e49d8b Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Wed, 1 Jul 2026 21:43:02 +0200 Subject: [PATCH 04/12] feat(federation): signed /api/federation/epoch endpoint + caller --- docs/systems/api.md | 3 + docs/systems/federation.md | 5 + packages/server/src/routes/federation.ts | 49 ++++ .../server/src/utils/federationEpoch.test.ts | 240 +++++++++++++++++- packages/server/src/utils/federationEpoch.ts | 64 +++++ 5 files changed, 360 insertions(+), 1 deletion(-) diff --git a/docs/systems/api.md b/docs/systems/api.md index a8091ef7..2485a2c6 100644 --- a/docs/systems/api.md +++ b/docs/systems/api.md @@ -324,12 +324,15 @@ DELETE /federation/peers/:id (admin) POST /federation/relay (HMAC-signed S2S) FederationRelayRequest → { accepted[], rejected[] } POST /federation/sync (HMAC-signed S2S) { sinceTimestamp, limit?, dmChannelId?, federatedId?, contextType? } → { events[], hasMore, checkpoint } POST /federation/users/lookup (HMAC-signed S2S, rate-limited 60/min/peer) { username } → { found, user? } +POST /federation/epoch (HMAC-signed S2S, HMAC-signed response) {} → { instanceId } ``` **`POST /api/federation/peer/accept`** — public, IP-rate-limited. Optional `approvalToken` (64-hex) on the request body proves mutual admin approval; required to promote an `awaiting_approval` row to `active` when the receiver has `autoAcceptPeering=0`. The receiver returns it in the 202 body when queueing the request for admin review (`{ queued: true, message, approvalToken }`); the initiator stores it and the receiver's `/approve` later forwards it back. See `federation.md` §1 "Approval Token Verification" for the full lifecycle and threat model. **`POST /api/federation/users/lookup`** — HMAC-authenticated S2S endpoint. Resolves a username on this instance to its canonical `(homeUserId, profile snapshot)`. Used by the cross-instance friend-add flow on the sender's home server before queuing a `friend_request_create` event. Responds to native, non-deleted users only; ignores `discoverable`. Returns `{ found: false, code: 'user_not_found' }` for stubs, tombstoned users, or unknown handles. See `federation.md` §1 "S2S User Lookup" for the full contract. +**`POST /api/federation/epoch`** — HMAC-authenticated S2S endpoint returning this instance's persistent epoch (`{ instanceId }`). The **request** is HMAC-signed (only a peer holding the shared secret may call it; unknown/revoked peers → 403, bad signature → 401, missing headers → 400) **and the response body is HMAC-signed** with the same secret (`X-Federation-Signature/Timestamp/Nonce` response headers), so the caller can verify the epoch before writing it as the peer's trusted baseline (`federation_peers.peer_instance_id`). The value is already public via `/instance/info`; signing is for baseline-integrity, not confidentiality. Caller: `fetchPeerEpoch(peer)` (`utils/federationEpoch.ts`), which fails safe — 404 (not-yet-upgraded peer), bad/absent response signature, or network/timeout all return `null` (retry next tick). Populates the epoch baseline deterministically via the bounded periodic epoch-refresh. See `federation.md` "Instance Epoch" §3.2. + ### Federation Peering Approval Queue Inbound + outbound peering approval queue (`autoAcceptPeering=0`). See [federation.md → Peer Approval Queue](federation.md#peer-approval-queue) and [federation.md → Outbound Peering Gate](federation.md#outbound-peering-gate). diff --git a/docs/systems/federation.md b/docs/systems/federation.md index ec9121d2..dd9d0236 100644 --- a/docs/systems/federation.md +++ b/docs/systems/federation.md @@ -307,6 +307,11 @@ Admin-initiated paths (`/peer/initiate`, `/approve`) do NOT call `ensurePeered`. | `/api/federation/peer/denied` | POST | HMAC | Receive denial notification for awaiting_approval peer | | `/api/federation/identity` | DELETE | HMAC | Delete federated user identity (soft/full mode) | | `/api/federation/users/lookup` | POST | HMAC, rate-limited 60/min/peer | Resolve a username on this instance to (homeUserId, profile snapshot) for cross-instance friend-request originators | +| `/api/federation/epoch` | POST | HMAC (signed request **and** signed response) | Return this instance's persistent epoch `{ instanceId }`; populates a peer's trusted epoch baseline (`peer_instance_id`) | + +### S2S Epoch Refresh (`POST /api/federation/epoch`) + +HMAC-authenticated in **both directions**: the request is signed (only a peer holding the shared secret may call it — unknown/revoked peers → 403, bad signature → 401, missing headers → 400) **and the response body `{ instanceId }` is HMAC-signed** with the same secret (`X-Federation-Signature/Timestamp/Nonce` response headers). The caller (`fetchPeerEpoch(peer)` in `utils/federationEpoch.ts`) verifies that response signature with the same secret before trusting the value, then writes it to `federation_peers.peer_instance_id`. Response-signing (not TLS-only) is deliberate: a poisoned baseline could drive a spurious data-heal on a live peer, so the newly-trusted epoch is authenticated (design §9). `fetchPeerEpoch` **fails safe** — a `404` from a not-yet-upgraded peer, an absent/invalid response signature, or a network/timeout error all return `null` (10s timeout via `AbortSignal.timeout`); the caller treats `null` as "retry on the next tick," never as an error to surface. This is the deterministic populator of the epoch baseline (the bounded periodic epoch-refresh, design §3.2), independent of organic relay traffic. ### S2S Identity Deletion (`DELETE /api/federation/identity`) diff --git a/packages/server/src/routes/federation.ts b/packages/server/src/routes/federation.ts index 378537a1..7e7c6661 100644 --- a/packages/server/src/routes/federation.ts +++ b/packages/server/src/routes/federation.ts @@ -2317,6 +2317,55 @@ export async function federationRoutes(app: FastifyInstance): Promise { }, ); + // ─── POST /api/federation/epoch ──────────────────────────────────────────── + // Server-to-server: return this instance's persistent epoch (instance_id). + // Authenticated via HMAC-SHA256 signature on the REQUEST (only a peer holding + // the shared secret may call it), and the RESPONSE body is HMAC-SIGNED with + // the same secret so the caller can verify the epoch it newly trusts before + // writing it as the peer's baseline (design §3.2 / §9). The value itself + // (instanceId) is already public via /instance/info; signing is for + // baseline-integrity, not confidentiality. + app.post( + '/api/federation/epoch', + { bodyLimit: 4 * 1024 }, + async (request, reply) => { + const db = getDb(); + + // 1. Parse and require federation headers (mirror relay/users-lookup). + const fedHeaders = parseFederationHeaders(request.headers as Record); + if (!fedHeaders) { + return reply.code(400).send({ error: 'Missing or malformed federation headers', statusCode: 400 }); + } + + // 2. Resolve the peer by origin. Reject unknown or revoked peers. + const peer = db + .select() + .from(schema.federationPeers) + .where(eq(schema.federationPeers.origin, fedHeaders.origin)) + .get(); + if (!peer || peer.status === 'revoked') { + return reply.code(403).send({ error: 'Not peered', statusCode: 403 }); + } + + // 3. Verify the inbound request signature (honours rotation grace). + const bodyString = JSON.stringify(request.body ?? {}); + if (!verifyPeerSignature(bodyString, fedHeaders.signature, fedHeaders.timestamp, fedHeaders.nonce, peer)) { + return reply.code(401).send({ error: 'Invalid signature', statusCode: 401 }); + } + + // 4. Sign the response body with the peer's shared secret and return it. + const responseBody = JSON.stringify({ instanceId: getInstanceId() }); + const sigHeaders = buildFederationHeaders(responseBody, peer.hmacSecret, getOurOrigin()); + reply.headers({ + 'X-Federation-Signature': sigHeaders['X-Federation-Signature'], + 'X-Federation-Timestamp': sigHeaders['X-Federation-Timestamp'], + 'X-Federation-Nonce': sigHeaders['X-Federation-Nonce'], + 'Content-Type': 'application/json', + }); + return reply.code(200).send(responseBody); + }, + ); + // ─── POST /api/federation/users/lookup ───────────────────────────────────── // Server-to-server: resolve a username on this instance to its canonical // (homeUserId, profile snapshot). Used by another instance to construct a diff --git a/packages/server/src/utils/federationEpoch.test.ts b/packages/server/src/utils/federationEpoch.test.ts index 733a0187..a0079f6d 100644 --- a/packages/server/src/utils/federationEpoch.test.ts +++ b/packages/server/src/utils/federationEpoch.test.ts @@ -1,10 +1,15 @@ -import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import Fastify, { type FastifyInstance } from 'fastify'; import Database from 'better-sqlite3'; import { drizzle } from 'drizzle-orm/better-sqlite3'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import * as schema from '../db/schema.js'; +import { setWorkerId } from './snowflake.js'; +import { buildFederationHeaders, verifySignature } from './federationAuth.js'; + +setWorkerId(1); const __dirname = path.dirname(fileURLToPath(import.meta.url)); let sqlite: Database.Database; @@ -16,6 +21,33 @@ vi.mock('../db/index.js', () => ({ schema, })); +vi.mock('../utils/auth.js', () => ({ + authenticate: async (req: { userId?: string }) => { + req.userId = 'admin-user'; + }, + requireAdmin: async () => { + // epoch endpoint is HMAC-authenticated, not JWT + }, +})); + +vi.mock('../ws/handler.js', () => ({ + connectionManager: { + sendToAdmins: vi.fn(), + getAllOnlineUserIds: () => [], + sendToUser: vi.fn(), + sendToDmMembers: vi.fn(), + }, +})); + +vi.mock('../utils/federationPeerActivation.js', () => ({ + onPeerActivated: vi.fn(async () => undefined), + onPeerDeactivated: vi.fn(async () => undefined), +})); + +const LOCAL_EPOCH = 'local-epoch-abcd'; +const PEER_ORIGIN = 'https://remote.example'; +const PEER_SECRET = 'peer-shared-secret-0123456789abcdef'; + function applyMigrations(db: Database.Database): void { const dir = path.resolve(__dirname, '../../drizzle'); for (const f of fs.readdirSync(dir).filter(f => f.endsWith('.sql')).sort()) { @@ -82,3 +114,209 @@ describe('getInstanceId', () => { expect(() => getInstanceId()).toThrow(/instance_id is not set/); }); }); + +function seedInstanceSettings(instanceId: string): void { + testDb.insert(schema.instanceSettings).values({ + id: 1, + instanceName: 'Local Backspace', + instanceId, + autoAcceptPeering: 1, + registrationOpen: 1, + updatedAt: Date.now(), + } as typeof schema.instanceSettings.$inferInsert).run(); +} + +function seedActivePeer(): void { + testDb.insert(schema.federationPeers).values({ + id: 'peer-remote', + origin: PEER_ORIGIN, + hmacSecret: PEER_SECRET, + status: 'active', + createdAt: Date.now(), + }).run(); +} + +async function buildApp(): Promise { + const app = Fastify({ logger: false }); + const { federationRoutes } = await import('../routes/federation.js'); + await app.register(federationRoutes); + await app.ready(); + return app; +} + +describe('POST /api/federation/epoch — signed request + signed response', () => { + // The module-level beforeEach already created a fresh in-memory DB and reset + // the instance-id cache; here we only seed rows and build the app. + let app: FastifyInstance; + + beforeEach(async () => { + seedInstanceSettings(LOCAL_EPOCH); + seedActivePeer(); + app = await buildApp(); + }); + + afterEach(async () => { + await app.close(); + vi.restoreAllMocks(); + }); + + it('returns 200 with a signed { instanceId } for a validly-signed request', async () => { + const body = JSON.stringify({}); + const headers = buildFederationHeaders(body, PEER_SECRET, PEER_ORIGIN); + + const response = await app.inject({ + method: 'POST', + url: '/api/federation/epoch', + headers, + payload: body, + }); + + expect(response.statusCode).toBe(200); + const parsed = response.json() as { instanceId?: string }; + expect(parsed.instanceId).toBe(LOCAL_EPOCH); + + // The response body must be HMAC-signed with the peer's shared secret. + const sigHeader = response.headers['x-federation-signature'] as string | undefined; + const tsHeader = response.headers['x-federation-timestamp'] as string | undefined; + const nonceHeader = response.headers['x-federation-nonce'] as string | undefined; + expect(sigHeader).toMatch(/^sha256=/); + expect(tsHeader).toBeTruthy(); + expect(nonceHeader).toBeTruthy(); + + const sig = (sigHeader ?? '').replace(/^sha256=/, ''); + const ts = Number(tsHeader); + const ok = verifySignature(response.body, sig, PEER_SECRET, ts, nonceHeader ?? null); + expect(ok).toBe(true); + }); + + it('returns 400 when federation headers are missing', async () => { + const response = await app.inject({ + method: 'POST', + url: '/api/federation/epoch', + payload: JSON.stringify({}), + headers: { 'content-type': 'application/json' }, + }); + + expect(response.statusCode).toBe(400); + }); + + it('returns 401 when the request is signed with the wrong secret', async () => { + const body = JSON.stringify({}); + const headers = buildFederationHeaders(body, 'the-wrong-secret', PEER_ORIGIN); + + const response = await app.inject({ + method: 'POST', + url: '/api/federation/epoch', + headers, + payload: body, + }); + + expect(response.statusCode).toBe(401); + }); + + it('returns 403 for an origin that is not a known peer', async () => { + const body = JSON.stringify({}); + const headers = buildFederationHeaders(body, PEER_SECRET, 'https://stranger.example'); + + const response = await app.inject({ + method: 'POST', + url: '/api/federation/epoch', + headers, + payload: body, + }); + + expect(response.statusCode).toBe(403); + }); + + it('returns 403 for a revoked peer', async () => { + testDb.insert(schema.federationPeers).values({ + id: 'peer-revoked', + origin: 'https://revoked.example', + hmacSecret: 'revoked-secret', + status: 'revoked', + createdAt: Date.now(), + }).run(); + + const body = JSON.stringify({}); + const headers = buildFederationHeaders(body, 'revoked-secret', 'https://revoked.example'); + + const response = await app.inject({ + method: 'POST', + url: '/api/federation/epoch', + headers, + payload: body, + }); + + expect(response.statusCode).toBe(403); + }); +}); + +describe('fetchPeerEpoch — signs request, verifies signed response, fails safe', () => { + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + function signedEpochResponse(instanceId: string, secret: string): Response { + const responseBody = JSON.stringify({ instanceId }); + // buildFederationHeaders returns a complete Record (signature, + // timestamp, nonce, origin, content-type) — exactly what the real handler sets. + const sigHeaders = buildFederationHeaders(responseBody, secret, PEER_ORIGIN); + return new Response(responseBody, { status: 200, headers: sigHeaders }); + } + + it('returns the instanceId when the response signature is valid', async () => { + vi.stubGlobal('fetch', vi.fn(async () => signedEpochResponse('remote-epoch-1', PEER_SECRET))); + const { fetchPeerEpoch } = await import('./federationEpoch.js'); + const result = await fetchPeerEpoch({ origin: PEER_ORIGIN, hmacSecret: PEER_SECRET }); + expect(result).toBe('remote-epoch-1'); + }); + + it('signs the outbound request with the peer secret', async () => { + const fetchMock = vi.fn(async () => signedEpochResponse('remote-epoch-1', PEER_SECRET)); + vi.stubGlobal('fetch', fetchMock); + const { fetchPeerEpoch } = await import('./federationEpoch.js'); + await fetchPeerEpoch({ origin: PEER_ORIGIN, hmacSecret: PEER_SECRET }); + + const call = fetchMock.mock.calls[0] as unknown as [string, RequestInit]; + expect(call[0]).toBe(`${PEER_ORIGIN}/api/federation/epoch`); + const sentHeaders = call[1].headers as Record; + const sig = (sentHeaders['X-Federation-Signature'] ?? '').replace(/^sha256=/, ''); + const ts = Number(sentHeaders['X-Federation-Timestamp']); + const nonce = sentHeaders['X-Federation-Nonce'] ?? null; + expect(verifySignature(call[1].body as string, sig, PEER_SECRET, ts, nonce)).toBe(true); + }); + + it('returns null when the response signature is invalid (wrong secret)', async () => { + vi.stubGlobal('fetch', vi.fn(async () => signedEpochResponse('remote-epoch-1', 'a-different-secret'))); + const { fetchPeerEpoch } = await import('./federationEpoch.js'); + const result = await fetchPeerEpoch({ origin: PEER_ORIGIN, hmacSecret: PEER_SECRET }); + expect(result).toBeNull(); + }); + + it('returns null on 404 (peer not yet upgraded)', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response('Not found', { status: 404 }))); + const { fetchPeerEpoch } = await import('./federationEpoch.js'); + const result = await fetchPeerEpoch({ origin: PEER_ORIGIN, hmacSecret: PEER_SECRET }); + expect(result).toBeNull(); + }); + + it('returns null on a network error (no throw escapes)', async () => { + vi.stubGlobal('fetch', vi.fn(async () => { throw new Error('ECONNREFUSED'); })); + const { fetchPeerEpoch } = await import('./federationEpoch.js'); + const result = await fetchPeerEpoch({ origin: PEER_ORIGIN, hmacSecret: PEER_SECRET }); + expect(result).toBeNull(); + }); + + it('returns null when the response omits the signature header', async () => { + vi.stubGlobal('fetch', vi.fn(async () => + new Response(JSON.stringify({ instanceId: 'remote-epoch-1' }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + )); + const { fetchPeerEpoch } = await import('./federationEpoch.js'); + const result = await fetchPeerEpoch({ origin: PEER_ORIGIN, hmacSecret: PEER_SECRET }); + expect(result).toBeNull(); + }); +}); diff --git a/packages/server/src/utils/federationEpoch.ts b/packages/server/src/utils/federationEpoch.ts index daa18834..5f35d0b0 100644 --- a/packages/server/src/utils/federationEpoch.ts +++ b/packages/server/src/utils/federationEpoch.ts @@ -1,5 +1,6 @@ import { eq } from 'drizzle-orm'; import { getDb, schema } from '../db/index.js'; +import { buildFederationHeaders, verifySignature, getOurOrigin } from './federationAuth.js'; let cached: string | null = null; @@ -22,3 +23,66 @@ export function getInstanceId(): string { export function __resetInstanceIdCacheForTest(): void { cached = null; } + +/** The minimal peer shape `fetchPeerEpoch` needs: its origin and our shared secret with it. */ +export interface PeerForEpoch { + origin: string; + hmacSecret: string; +} + +/** + * Fetch a peer's authenticated instance epoch via `POST /api/federation/epoch`. + * + * The request is HMAC-signed with the shared secret (so only an established + * peer can make the call), and the peer's response body is HMAC-verified with + * the same secret before its value is trusted — a poisoned baseline can drive a + * spurious heal on a live peer (design §9), so the epoch we newly trust is + * signed, not TLS-only. + * + * Fails safe: any failure — a 404 from a not-yet-upgraded peer, a bad/absent + * response signature, or a network/timeout error — returns `null`. Callers + * treat `null` as "retry on the next tick," never as an error to surface. No + * exception escapes this function. + */ +export async function fetchPeerEpoch(peer: PeerForEpoch): Promise { + const body = JSON.stringify({}); + const headers = buildFederationHeaders(body, peer.hmacSecret, getOurOrigin()); + + let res: Response; + try { + res = await fetch(`${peer.origin}/api/federation/epoch`, { + method: 'POST', + headers, + body, + signal: AbortSignal.timeout(10000), + }); + } catch { + // Network error / timeout — benign no-op, retry later. + return null; + } + + // 404 = peer not yet upgraded (endpoint absent); any other non-2xx = error. + if (res.status === 404 || !res.ok) return null; + + let text: string; + try { + text = await res.text(); + } catch { + return null; + } + + // Verify the response signature with the SAME secret and arg order the peer's + // handler signed it with. A mismatch means we must not trust the value. + const sig = (res.headers.get('x-federation-signature') ?? '').replace(/^sha256=/, ''); + const ts = Number(res.headers.get('x-federation-timestamp')); + const nonce = res.headers.get('x-federation-nonce'); + if (!sig || !Number.isFinite(ts) || !verifySignature(text, sig, peer.hmacSecret, ts, nonce)) { + return null; + } + + try { + return (JSON.parse(text) as { instanceId?: string }).instanceId ?? null; + } catch { + return null; + } +} From 8f60e92f9455d4fa386e8a95742c65c74f4816a5 Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Wed, 1 Jul 2026 21:49:18 +0200 Subject: [PATCH 05/12] feat(federation): deterministic baseline epoch-refresh worker --- docs/systems/federation.md | 3 + .../server/src/utils/federationEpoch.test.ts | 91 +++++++++++++++++++ packages/server/src/utils/federationEpoch.ts | 51 ++++++++++- packages/server/src/utils/federationWorker.ts | 15 +++ 4 files changed, 159 insertions(+), 1 deletion(-) diff --git a/docs/systems/federation.md b/docs/systems/federation.md index dd9d0236..47751731 100644 --- a/docs/systems/federation.md +++ b/docs/systems/federation.md @@ -313,6 +313,8 @@ Admin-initiated paths (`/peer/initiate`, `/approve`) do NOT call `ensurePeered`. HMAC-authenticated in **both directions**: the request is signed (only a peer holding the shared secret may call it — unknown/revoked peers → 403, bad signature → 401, missing headers → 400) **and the response body `{ instanceId }` is HMAC-signed** with the same secret (`X-Federation-Signature/Timestamp/Nonce` response headers). The caller (`fetchPeerEpoch(peer)` in `utils/federationEpoch.ts`) verifies that response signature with the same secret before trusting the value, then writes it to `federation_peers.peer_instance_id`. Response-signing (not TLS-only) is deliberate: a poisoned baseline could drive a spurious data-heal on a live peer, so the newly-trusted epoch is authenticated (design §9). `fetchPeerEpoch` **fails safe** — a `404` from a not-yet-upgraded peer, an absent/invalid response signature, or a network/timeout error all return `null` (10s timeout via `AbortSignal.timeout`); the caller treats `null` as "retry on the next tick," never as an error to surface. This is the deterministic populator of the epoch baseline (the bounded periodic epoch-refresh, design §3.2), independent of organic relay traffic. +**Deterministic epoch-refresh driver (`refreshPeerEpochs()` in `utils/federationEpoch.ts`).** Selects every `active` peer whose `peer_instance_id IS NULL`, calls `fetchPeerEpoch(peer)` once each, and on a non-null result writes the epoch via `UPDATE ... SET peer_instance_id WHERE id = ? AND peer_instance_id IS NULL`. The trailing `IS NULL` guard makes it **populate-if-null only** — it can never overwrite a baseline another path (relay envelope, handshake) already established — and makes it **self-terminating**: once a peer's `peer_instance_id` is set, the `IS NULL` filter excludes it, so it is never fetched again. A `null` from `fetchPeerEpoch` (404 / bad-sig / network) is a benign `continue` with no error log-spam, retried next tick. Wired into the federation worker in two places: once at `startFederationWorkers()` startup and once at the end of `processHealthCheckTick()` (the existing 15-minute health-check tick), both as `refreshPeerEpochs().catch(() => {})`. This guarantees the trusted baseline is populated within one refresh cycle of an upgrade, independent of user/relay activity — the load-bearing populator that relay-only population cannot cover for idle peers. + ### S2S Identity Deletion (`DELETE /api/federation/identity`) Allows a home instance to remove a user's replicated identity from a remote instance. @@ -1563,6 +1565,7 @@ All workers are started by `startFederationWorkers()` on server boot and stopped | Outbox delivery | 10s | 50 | 30s | `processOutboxTick` | | File download | 30s | 5 | 60s | `processFileQueueTick` | | Health check | 15min | all unreachable | 10s | `processHealthCheckTick` | +| Epoch-refresh baseline | Startup + 15min (end of health tick) | active peers w/ `peer_instance_id IS NULL` | 10s per peer | `refreshPeerEpochs` (populate-if-null, self-terminating) | | Janitor | 1h | -- | -- | `runFederationJanitor` (sync) | | Startup bootstrap sync | Once at startup | -- | 30s per page | `startupBootstrapSync` → `onPeerActivated` | diff --git a/packages/server/src/utils/federationEpoch.test.ts b/packages/server/src/utils/federationEpoch.test.ts index a0079f6d..1813eca7 100644 --- a/packages/server/src/utils/federationEpoch.test.ts +++ b/packages/server/src/utils/federationEpoch.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import Fastify, { type FastifyInstance } from 'fastify'; import Database from 'better-sqlite3'; import { drizzle } from 'drizzle-orm/better-sqlite3'; +import { eq } from 'drizzle-orm'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -320,3 +321,93 @@ describe('fetchPeerEpoch — signs request, verifies signed response, fails safe expect(result).toBeNull(); }); }); + +describe('refreshPeerEpochs — deterministic populate-if-null baseline (self-terminating)', () => { + // Drives the REAL refreshPeerEpochs → fetchPeerEpoch → verifySignature round-trip. + // fetchPeerEpoch is deliberately NOT stubbed: a signing/arg-order mismatch must + // fail these assertions loudly rather than degrade to a silent null (which would + // masquerade as a benign 404 and quietly disable the whole refresh). + beforeEach(() => { + // Local instance epoch must be readable (getOurOrigin does not need it, but the + // module is shared; seed for parity with real boot state). + seedInstanceSettings(LOCAL_EPOCH); + seedActivePeer(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + /** A response body signed with `secret` over exactly the bytes we return. */ + function signedEpochResponse(instanceId: string, secret: string): Response { + const responseBody = JSON.stringify({ instanceId }); + const sigHeaders = buildFederationHeaders(responseBody, secret, PEER_ORIGIN); + return new Response(responseBody, { status: 200, headers: sigHeaders }); + } + + function readPeerInstanceId(): string | null { + const row = testDb + .select({ peerInstanceId: schema.federationPeers.peerInstanceId }) + .from(schema.federationPeers) + .where(eq(schema.federationPeers.id, 'peer-remote')) + .get(); + return row?.peerInstanceId ?? null; + } + + it('populates peer_instance_id from a validly-signed response, then self-terminates', async () => { + const fetchMock = vi.fn(async () => signedEpochResponse('E1', PEER_SECRET)); + vi.stubGlobal('fetch', fetchMock); + + const { refreshPeerEpochs } = await import('./federationEpoch.js'); + await refreshPeerEpochs(); + + expect(readPeerInstanceId()).toBe('E1'); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // Second pass: the peer is now non-null, so the IS NULL filter excludes it — + // no further fetch is issued. Self-termination is structural, not incidental. + await refreshPeerEpochs(); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(readPeerInstanceId()).toBe('E1'); + }); + + it('leaves the baseline NULL when the response signature is invalid (tampered)', async () => { + // Signed with a different secret → verification fails → fetchPeerEpoch returns null. + const fetchMock = vi.fn(async () => signedEpochResponse('E1', 'a-different-secret')); + vi.stubGlobal('fetch', fetchMock); + + const { refreshPeerEpochs } = await import('./federationEpoch.js'); + await refreshPeerEpochs(); + + expect(readPeerInstanceId()).toBeNull(); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('leaves the baseline NULL and does not throw on a 404 (peer not yet upgraded)', async () => { + const fetchMock = vi.fn(async () => new Response('Not found', { status: 404 })); + vi.stubGlobal('fetch', fetchMock); + + const { refreshPeerEpochs } = await import('./federationEpoch.js'); + await expect(refreshPeerEpochs()).resolves.toBeUndefined(); + + expect(readPeerInstanceId()).toBeNull(); + }); + + it('never overwrites an already-populated baseline (populate-if-null only)', async () => { + testDb.update(schema.federationPeers) + .set({ peerInstanceId: 'pre-existing' }) + .where(eq(schema.federationPeers.id, 'peer-remote')) + .run(); + + const fetchMock = vi.fn(async () => signedEpochResponse('E1', PEER_SECRET)); + vi.stubGlobal('fetch', fetchMock); + + const { refreshPeerEpochs } = await import('./federationEpoch.js'); + await refreshPeerEpochs(); + + // Already non-null → excluded by the IS NULL filter → no fetch, value untouched. + expect(fetchMock).not.toHaveBeenCalled(); + expect(readPeerInstanceId()).toBe('pre-existing'); + }); +}); diff --git a/packages/server/src/utils/federationEpoch.ts b/packages/server/src/utils/federationEpoch.ts index 5f35d0b0..bbb82388 100644 --- a/packages/server/src/utils/federationEpoch.ts +++ b/packages/server/src/utils/federationEpoch.ts @@ -1,4 +1,4 @@ -import { eq } from 'drizzle-orm'; +import { and, eq, isNull } from 'drizzle-orm'; import { getDb, schema } from '../db/index.js'; import { buildFederationHeaders, verifySignature, getOurOrigin } from './federationAuth.js'; @@ -86,3 +86,52 @@ export async function fetchPeerEpoch(peer: PeerForEpoch): Promise return null; } } + +/** + * Deterministic baseline populator: for each `active` peer whose + * `peer_instance_id` is still NULL, fetch its authenticated epoch once and store + * it. This is the load-bearing guarantee (design §3.2) — it populates the + * trusted baseline within one refresh cycle of an upgrade, independent of any + * user/relay activity, closing the window that relay-only population leaves for + * idle peers. + * + * Populate-if-null ONLY: the `UPDATE ... WHERE peer_instance_id IS NULL` guard + * makes it structurally impossible to overwrite a baseline that another path + * (relay, handshake) already established. Self-terminating: once a peer's + * `peer_instance_id` is set, the `isNull` filter excludes it, so it is never + * fetched again. + * + * Staggered-rollout tolerant: `fetchPeerEpoch` returns `null` for a 404 + * (not-yet-upgraded peer), a bad/absent response signature, or a network error. + * All of those are benign no-ops — we simply skip the peer and retry on the next + * tick, with no error log-spam. No exception escapes this function. + */ +export async function refreshPeerEpochs(): Promise { + const db = getDb(); + const peers = db + .select({ + id: schema.federationPeers.id, + origin: schema.federationPeers.origin, + hmacSecret: schema.federationPeers.hmacSecret, + }) + .from(schema.federationPeers) + .where(and( + eq(schema.federationPeers.status, 'active'), + isNull(schema.federationPeers.peerInstanceId), + )) + .all(); + + for (const peer of peers) { + const epoch = await fetchPeerEpoch(peer); + if (!epoch) continue; // 404 / bad-sig / network → retry next tick, no log-spam. + + // Populate-if-null only: the IS NULL guard never overwrites a non-null baseline. + db.update(schema.federationPeers) + .set({ peerInstanceId: epoch }) + .where(and( + eq(schema.federationPeers.id, peer.id), + isNull(schema.federationPeers.peerInstanceId), + )) + .run(); + } +} diff --git a/packages/server/src/utils/federationWorker.ts b/packages/server/src/utils/federationWorker.ts index 674c55d5..32b9fc6a 100644 --- a/packages/server/src/utils/federationWorker.ts +++ b/packages/server/src/utils/federationWorker.ts @@ -15,6 +15,7 @@ import { startupBootstrapSync, onPeerDeactivated } from './federationPeerActivat import { probePeerReachable, markPeerRecovered } from './federationRecovery.js'; import { backfillReplicatedProfileAssets } from '../routes/federation.js'; import { invokePermanentFailureCallback } from './federationRollback.js'; +import { refreshPeerEpochs } from './federationEpoch.js'; import fs from 'node:fs'; import path from 'node:path'; import crypto from 'node:crypto'; @@ -1162,6 +1163,14 @@ async function processHealthCheckTick(): Promise { console.warn(`[federation-worker] Auto-rotation failed for peer ${peer.origin}: ${message}`); } } + + // ── Deterministic baseline epoch-refresh ──────────────────────────────────── + // Populate-if-null, self-terminating: fill peer_instance_id for active peers + // whose baseline is still NULL (design §3.2). Runs every tick so the baseline + // is established within one 15-minute cycle of an upgrade, independent of any + // relay/user activity. Best-effort — a failed fetch is a benign no-op retried + // next tick, so it never disturbs the rest of the health-check work. + await refreshPeerEpochs().catch(() => {}); } // ─── Federated Call Health Sweep ──────────────────────────────────────────── @@ -1235,6 +1244,12 @@ export function startFederationWorkers(): void { console.error('[federation-worker] federatedCallSentinel tick failed:', err) ); }, FEDERATED_CALL_SENTINEL_MS); + // Deterministic baseline epoch-refresh at startup (design §3.2): populate + // peer_instance_id for any active peer whose baseline is still NULL, so an + // instance that upgrades sees its peers' epochs within one cycle regardless of + // traffic. Best-effort, self-terminating (populate-if-null). + refreshPeerEpochs().catch(() => {}); + // Bootstrap sync for freshly-peered rows (async, non-blocking) startupBootstrapSync().catch((err) => { console.error('[federation-worker] Startup bootstrap sync error:', err); From 3b1a0b64a3c7d40b48d4952629d3a5f90f1c0261 Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Wed, 1 Jul 2026 21:58:03 +0200 Subject: [PATCH 06/12] feat(federation): relay envelope populates peer epoch baseline --- docs/systems/federation.md | 2 + packages/server/src/routes/federation.ts | 19 +++++ .../server/src/utils/federationEpoch.test.ts | 81 +++++++++++++++++++ .../server/src/utils/federationWorker.test.ts | 18 +++++ packages/server/src/utils/federationWorker.ts | 7 +- 5 files changed, 126 insertions(+), 1 deletion(-) diff --git a/docs/systems/federation.md b/docs/systems/federation.md index 47751731..655f22f2 100644 --- a/docs/systems/federation.md +++ b/docs/systems/federation.md @@ -406,6 +406,8 @@ Two layers of replay protection: **Important:** The body is re-serialized server-side. This means Fastify's JSON parsing and re-stringification must produce identical output to the sender's `JSON.stringify`. In practice this works because both sides use standard `JSON.stringify` with no custom replacers. +**Relay-envelope epoch (fast-path baseline population, design §3.2).** `FederationRelayRequest` carries `sourceInstanceId?: string` — the sender stamps its current epoch (`getInstanceId()`) when building the request in `federationWorker.ts`. Because the whole body is HMAC-verified above (step 4), a valid relay authentically carries the sender's current incarnation id. Immediately after the signature check passes (and only there — the authenticated boundary), the receiver runs **populate-if-null**: `if (sourceInstanceId && peer.peerInstanceId IS NULL) UPDATE federation_peers SET peer_instance_id = WHERE id = ? AND peer_instance_id IS NULL`. This is the *fast-path* baseline populator — it fills the trusted epoch the instant organic traffic flows, usually before the deterministic 15-minute `refreshPeerEpochs` backstop fires. It **never overwrites** a non-null baseline: a differing incarnation implies a different HMAC secret that would have failed verification, so a valid relay can never carry an epoch differing from an established baseline. Runs independent of per-event processing and does not affect relay accept/reject. Backward-compatible: older peers omit `sourceInstanceId` → the update is skipped (no-op). + --- ## 3. Identity Resolution diff --git a/packages/server/src/routes/federation.ts b/packages/server/src/routes/federation.ts index 7e7c6661..40fc2152 100644 --- a/packages/server/src/routes/federation.ts +++ b/packages/server/src/routes/federation.ts @@ -2259,6 +2259,25 @@ export async function federationRoutes(app: FastifyInstance): Promise { return reply.code(401).send({ error: 'Invalid signature', statusCode: 401 }); } + // 1b-epoch. Fast-path baseline population (design §3.2). The signature just + // verified proves the peer holds the current shared secret, so the epoch it + // carries in `sourceInstanceId` is authentic. Populate-if-null ONLY: a valid + // relay can never carry an epoch differing from a non-null baseline (a + // different incarnation implies a different secret that fails HMAC), so we + // only ever fill a NULL — never overwrite. This is independent of per-event + // processing and does not affect relay accept/reject in any way. Old peers + // omit the field → skip (backward-compatible no-op). + const claimedEpoch = request.body.sourceInstanceId; + if (claimedEpoch && !peer.peerInstanceId) { + db.update(schema.federationPeers) + .set({ peerInstanceId: claimedEpoch }) + .where(and( + eq(schema.federationPeers.id, peer.id), + isNull(schema.federationPeers.peerInstanceId), + )) + .run(); + } + // 1c. Nonce-based replay protection if (fedHeaders.nonce) { if (isNonceDuplicate(peer.origin, fedHeaders.nonce)) { diff --git a/packages/server/src/utils/federationEpoch.test.ts b/packages/server/src/utils/federationEpoch.test.ts index 1813eca7..ccdea5cb 100644 --- a/packages/server/src/utils/federationEpoch.test.ts +++ b/packages/server/src/utils/federationEpoch.test.ts @@ -411,3 +411,84 @@ describe('refreshPeerEpochs — deterministic populate-if-null baseline (self-te expect(readPeerInstanceId()).toBe('pre-existing'); }); }); + +describe('POST /api/federation/relay — fast-path epoch baseline (populate-if-null)', () => { + // A verified inbound relay authentically carries the sender's current epoch in + // `sourceInstanceId` (design §3.2). On the authenticated path only, the receiver + // fills a NULL `peer_instance_id` — never overwrites a non-null baseline. + let app: FastifyInstance; + + beforeEach(async () => { + seedInstanceSettings(LOCAL_EPOCH); + seedActivePeer(); + app = await buildApp(); + }); + + afterEach(async () => { + await app.close(); + vi.restoreAllMocks(); + }); + + function readPeerInstanceId(): string | null { + const row = testDb + .select({ peerInstanceId: schema.federationPeers.peerInstanceId }) + .from(schema.federationPeers) + .where(eq(schema.federationPeers.id, 'peer-remote')) + .get(); + return row?.peerInstanceId ?? null; + } + + /** Send a validly-signed relay (empty event batch) carrying `sourceInstanceId`. */ + async function injectSignedRelay(sourceInstanceId?: string): Promise { + const relay: Record = { + version: 1, + sourceInstance: PEER_ORIGIN, + events: [], + }; + if (sourceInstanceId !== undefined) relay.sourceInstanceId = sourceInstanceId; + const body = JSON.stringify(relay); + const headers = buildFederationHeaders(body, PEER_SECRET, PEER_ORIGIN); + const response = await app.inject({ + method: 'POST', + url: '/api/federation/relay', + headers, + payload: body, + }); + return response.statusCode; + } + + it('populates a NULL baseline from the epoch a verified relay carries', async () => { + expect(readPeerInstanceId()).toBeNull(); + const status = await injectSignedRelay('remote-epoch-A'); + expect(status).toBe(200); + expect(readPeerInstanceId()).toBe('remote-epoch-A'); + }); + + it('never overwrites a non-null baseline (a valid relay cannot carry a differing epoch)', async () => { + const first = await injectSignedRelay('remote-epoch-A'); + expect(first).toBe(200); + expect(readPeerInstanceId()).toBe('remote-epoch-A'); + + // A subsequent relay claiming a different epoch must leave the baseline intact. + const second = await injectSignedRelay('remote-epoch-B'); + expect(second).toBe(200); + expect(readPeerInstanceId()).toBe('remote-epoch-A'); + }); + + it('is a no-op when a pre-existing baseline is already set', async () => { + testDb.update(schema.federationPeers) + .set({ peerInstanceId: 'pre-existing' }) + .where(eq(schema.federationPeers.id, 'peer-remote')) + .run(); + + const status = await injectSignedRelay('remote-epoch-A'); + expect(status).toBe(200); + expect(readPeerInstanceId()).toBe('pre-existing'); + }); + + it('is a no-op for a backward-compatible relay that omits sourceInstanceId', async () => { + const status = await injectSignedRelay(undefined); + expect(status).toBe(200); + expect(readPeerInstanceId()).toBeNull(); + }); +}); diff --git a/packages/server/src/utils/federationWorker.test.ts b/packages/server/src/utils/federationWorker.test.ts index d0f4e3a8..fb871930 100644 --- a/packages/server/src/utils/federationWorker.test.ts +++ b/packages/server/src/utils/federationWorker.test.ts @@ -6,6 +6,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import * as schema from '../db/schema.js'; import { eq } from 'drizzle-orm'; +import { __resetInstanceIdCacheForTest } from './federationEpoch.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); type TestDb = ReturnType>; @@ -84,6 +85,20 @@ function applyMigrations(db: Database.Database): void { } } +/** + * Seed this instance's epoch so the outbox relay builder can stamp + * `sourceInstanceId` via getInstanceId(). Resets the module cache so the fresh + * per-test DB row is read rather than a value cached from a prior test. + */ +function seedInstanceEpoch(): void { + testDb.insert(schema.instanceSettings).values({ + id: 1, + instanceId: 'worker-test-epoch', + updatedAt: Date.now(), + } as typeof schema.instanceSettings.$inferInsert).run(); + __resetInstanceIdCacheForTest(); +} + function seedPeer(id: string): void { testDb.insert(schema.federationPeers).values({ id, origin: 'https://peer.example', hmacSecret: 'secret', @@ -108,6 +123,7 @@ describe('outbox worker — duplicate rejection is terminal', () => { sqlite = new Database(':memory:'); testDb = drizzle(sqlite, { schema }); applyMigrations(sqlite); + seedInstanceEpoch(); vi.restoreAllMocks(); // Re-apply the static mocks that vi.restoreAllMocks() would undo. // isFederationRelayEnabled is mocked at module level via vi.mock (hoisted), @@ -303,6 +319,7 @@ describe('outbox worker — terminal rejection reasons + rollback invocation', ( sqlite = new Database(':memory:'); testDb = drizzle(sqlite, { schema }); applyMigrations(sqlite); + seedInstanceEpoch(); vi.restoreAllMocks(); invokeRollbackMock.mockReset(); }); @@ -431,6 +448,7 @@ describe('unreachable transition resets probe pacing', () => { sqlite = new Database(':memory:'); testDb = drizzle(sqlite, { schema }); applyMigrations(sqlite); + seedInstanceEpoch(); vi.restoreAllMocks(); }); diff --git a/packages/server/src/utils/federationWorker.ts b/packages/server/src/utils/federationWorker.ts index 32b9fc6a..7d6ffc8d 100644 --- a/packages/server/src/utils/federationWorker.ts +++ b/packages/server/src/utils/federationWorker.ts @@ -15,7 +15,7 @@ import { startupBootstrapSync, onPeerDeactivated } from './federationPeerActivat import { probePeerReachable, markPeerRecovered } from './federationRecovery.js'; import { backfillReplicatedProfileAssets } from '../routes/federation.js'; import { invokePermanentFailureCallback } from './federationRollback.js'; -import { refreshPeerEpochs } from './federationEpoch.js'; +import { refreshPeerEpochs, getInstanceId } from './federationEpoch.js'; import fs from 'node:fs'; import path from 'node:path'; import crypto from 'node:crypto'; @@ -235,6 +235,11 @@ export async function processOutboxTick(): Promise { const request: FederationRelayRequest = { version: 1, sourceInstance: ourOrigin, + // Stamp our current epoch so a verified relay authentically carries this + // instance's incarnation id — the receiver uses it as the fast-path + // populate-if-null baseline (design §3.2). A reset instance cannot sign a + // valid relay, so this never carries a *new* epoch post-reset. + sourceInstanceId: getInstanceId(), events, }; From 45e1c88bdca7241d02060f4666a8c58afd9a3ecf Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Wed, 1 Jul 2026 22:09:43 +0200 Subject: [PATCH 07/12] feat(federation): reset detection (markPeerReset) via handshake + probe --- docs/systems/federation.md | 19 ++ docs/systems/websocket.md | 1 + packages/server/src/routes/federation.ts | 25 ++- .../src/utils/federationRecovery.test.ts | 89 +++++++++- .../server/src/utils/federationRecovery.ts | 66 ++++++- .../server/src/utils/federationReset.test.ts | 163 ++++++++++++++++++ packages/server/src/utils/federationReset.ts | 148 ++++++++++++++++ packages/server/src/utils/federationWorker.ts | 14 +- packages/shared/src/types.ts | 1 + 9 files changed, 506 insertions(+), 20 deletions(-) create mode 100644 packages/server/src/utils/federationReset.test.ts create mode 100644 packages/server/src/utils/federationReset.ts diff --git a/docs/systems/federation.md b/docs/systems/federation.md index 655f22f2..d44b5ab9 100644 --- a/docs/systems/federation.md +++ b/docs/systems/federation.md @@ -315,6 +315,24 @@ HMAC-authenticated in **both directions**: the request is signed (only a peer ho **Deterministic epoch-refresh driver (`refreshPeerEpochs()` in `utils/federationEpoch.ts`).** Selects every `active` peer whose `peer_instance_id IS NULL`, calls `fetchPeerEpoch(peer)` once each, and on a non-null result writes the epoch via `UPDATE ... SET peer_instance_id WHERE id = ? AND peer_instance_id IS NULL`. The trailing `IS NULL` guard makes it **populate-if-null only** — it can never overwrite a baseline another path (relay envelope, handshake) already established — and makes it **self-terminating**: once a peer's `peer_instance_id` is set, the `IS NULL` filter excludes it, so it is never fetched again. A `null` from `fetchPeerEpoch` (404 / bad-sig / network) is a benign `continue` with no error log-spam, retried next tick. Wired into the federation worker in two places: once at `startFederationWorkers()` startup and once at the end of `processHealthCheckTick()` (the existing 15-minute health-check tick), both as `refreshPeerEpochs().catch(() => {})`. This guarantees the trusted baseline is populated within one refresh cycle of an upgrade, independent of user/relay activity — the load-bearing populator that relay-only population cannot cover for idle peers. +### Reset Detection (`markPeerReset` — `utils/federationReset.ts`) + +The epoch is a **detection signal only, never an authorization signal.** When a peer behind a known origin advertises an epoch that differs from the trusted baseline (`peer_instance_id`), the instance was wiped and a new incarnation stood up on the same domain. `markPeerReset(peerId, origin, deadEpoch, observedEpoch)` routes the peer for admin attention and snapshots the dead incarnation — but performs **NO rekey, NO tombstone, NO handle change, NO content deletion.** The actual data heal fires only later, from `onPeerActivated` after an admin-authenticated re-peer (design §6). Because detection grants no capability and destroys nothing, it is safe to fire on an unauthenticated signal: the worst a spoofed detection can do is flag a peer for admin review (admin-reversible nuisance). + +In a single transaction, `markPeerReset`: +1. Sets the peer `status='needs_attention'`, `needs_attention_reason='peer_reset_detected'`, and `observed_peer_instance_id=observedEpoch`. **`peer_instance_id` (the trusted baseline) and `hmac_secret` are left untouched** — an unauthenticated observation never rekeys trust; the observed-but-untrusted epoch lives only in `observed_peer_instance_id`. +2. **Snapshots the dead incarnation:** sets `users.federation_heal_pending = 1` for every non-deleted user whose `home_instance` matches the origin. The match keys on `extractDomain(origin)` (bare domain, the canonical `home_instance` form) and defensively also matches the `https://`/`http://`-prefixed forms so any legacy full-URL straggler is caught (`homeInstanceMatch()`). Any stub created *after* detection (e.g. a friend-add reaching the new incarnation directly) is un-flagged and survives the heal. +3. **Journals the dead incarnation durably** by upserting a `federation_reset_events` row keyed by origin: `{ dead_epoch=deadEpoch, new_epoch=NULL, detected_at, resolved_at=NULL, stub_count, orphaned_account_count }`. `stub_count` counts flagged pure S2S stubs (`password_hash = '!federation-replicated'`); `orphaned_account_count` counts flagged real accounts. This row survives the peer-row deletion that Re-peer performs, preserving `dead_epoch` for the false-positive guard (design §6.1) and the admin surface. +4. After the transaction, broadcasts `federation_peers_changed` and `federation_peer_reset_detected {origin}` to admins. + +**Idempotent / double-reset:** if an *unresolved* `federation_reset_events` row already exists for the origin (the peer reset again before an admin resolved the first), the original `dead_epoch` and `detected_at` are **preserved** (that is the incarnation whose users are already snapshotted) — only the summary counts are refreshed. `dead_epoch` is never overwritten on an unresolved row. A prior *resolved* reset starts a fresh journal entry. + +**Detection sources (both wired in this feature):** +- **Inbound handshake** — `/peer/accept` landing on an `active`/`needs_attention` row (`routes/federation.ts`): before the idempotent-200 return, `if (reqInstanceId && existing.peerInstanceId && reqInstanceId !== existing.peerInstanceId) markPeerReset(...)`. The guard still returns 200 and **still does not rekey** — the anti-hijack property is preserved verbatim; detection is layered on top. +- **Reachability probe** — `probePeerReachable()` (`utils/federationRecovery.ts`) now parses `instanceId` from the `/api/instance/info` response (returning `{ reachable, instanceId }`; a missing/unparseable epoch is `null`, never an error). The shared decision helper `recoverOrDetectReset(peer, result)` — used by both the background recovery tick (`processRecoveryTick`) and the manual recheck endpoint — routes to `markPeerReset` (returning `'reset_detected'`) when the peer has a non-null `peer_instance_id` and the probed epoch differs, and **does NOT call `markPeerRecovered`**. Rationale: a genuinely reset peer's HMAC secret is desynced, so flipping it back to `active` via a reachability probe would resume relay against a dead secret. Only when the probed epoch matches the baseline (or the baseline is null / epoch unknown) does the normal recovery-to-active path run. The manual recheck endpoint returns `{ recovered: false, status: 'needs_attention' }` on `'reset_detected'`. + +Legacy peers advertise no epoch (`instanceId` null), so detection requires a non-null observed epoch differing from a non-null stored baseline — legacy peers never trigger it, and the existing `auth_failures → needs_attention → manual Reset` path continues unchanged for them. + ### S2S Identity Deletion (`DELETE /api/federation/identity`) Allows a home instance to remove a user's replicated identity from a remote instance. @@ -357,6 +375,7 @@ These S→C events are pushed to the acting user's connected clients by the fede |-------|-------------|---------| | `federation_peer_rejected` | Outbox worker receives `403 PEERING_REQUIRES_APPROVAL` from a remote instance during auto-peering | `{ peerId: string, origin: string }` | | `federation_peer_active` | A previously `rejected` peer transitions to `active` (e.g., via manual `peer/initiate` or incoming `peer/accept`) | `{ peerId: string, origin: string }` | +| `federation_peer_reset_detected` | A peer's advertised instance epoch differs from the trusted baseline (wipe-and-reinstall on the same domain) — emitted by `markPeerReset` after routing the peer to `needs_attention` | `{ origin: string }` (admin-only, via `sendToAdmins`) | --- diff --git a/docs/systems/websocket.md b/docs/systems/websocket.md index bd0ae788..911be66c 100644 --- a/docs/systems/websocket.md +++ b/docs/systems/websocket.md @@ -194,6 +194,7 @@ reason: `'displaced'` (new tab) | `'session_closed'` |------|--------|-------| | `federation_file_rejected` | messageId, dmChannelId, attachmentId, affectedUsers[] | DM members | | `federation_approval_request_received` | — (refetch trigger; payload: `{ type }`) | admins. Fires for **both** inbound peering requests (remote → us) AND outbound queue creation when the [Outbound Peering Gate](federation.md#outbound-peering-gate) creates a `peer_approval_requests` row in response to a user_action. Payload shape unchanged from the inbound-only behavior; only the firing surface widened. | +| `federation_peer_reset_detected` | `{ origin: string }` | admins. Fires from `markPeerReset` when a peer's advertised instance epoch differs from the trusted baseline (a wipe-and-reinstall on the same domain — see [Reset Detection](federation.md#reset-detection-markpeerreset--utilsfederationresetts)). Detection-only: the peer was routed to `needs_attention` (reason `peer_reset_detected`) with no rekey/tombstone. Paired with a `federation_peers_changed` broadcast; client surfaces the reset for one-click Re-peer. | | `peering_subscription_changed` | — (refetch trigger; payload: `{ type }`) | the subscribing user (all of their connected sessions). Fires when a `peer_approval_subscribers` row belonging to the user is created, modified, or deleted (gate fan-in, user cancel, parent cascade). Client refetches `GET /api/federation/peering-subscriptions`. | | `peering_notification_received` | `{ type, kind: 'approved' \| 'denied' \| 'expired' }` | the user the notification belongs to. Fires when a `peer_approval_notifications` row is created (`onPeerActivated` outbound fanout, outbound `/deny` fanout, janitor outbound expiry). Client refetches `GET /api/federation/peering-notifications` and may surface a transient toast for online users. | diff --git a/packages/server/src/routes/federation.ts b/packages/server/src/routes/federation.ts index 40fc2152..de102c51 100644 --- a/packages/server/src/routes/federation.ts +++ b/packages/server/src/routes/federation.ts @@ -20,7 +20,8 @@ import { tombstoneUser, collectDeletionBroadcastTargets, collectProfileBroadcast import { computeFederatedId, getDmParticipants, sendCallRelay } from '../utils/federationOutbox.js'; import { onPeerActivated, onPeerDeactivated } from '../utils/federationPeerActivation.js'; import { getInstanceId } from '../utils/federationEpoch.js'; -import { probePeerReachable, markPeerRecovered } from '../utils/federationRecovery.js'; +import { probePeerReachable, recoverOrDetectReset } from '../utils/federationRecovery.js'; +import { markPeerReset } from '../utils/federationReset.js'; import { getDmMessageWithUser } from './dm.js'; import type { FederationRelayRequest, FederationRelayResponse, FederationRelayEvent, FederationRelayAttachment, FederationSyncRequest, FederationSyncResponse, DmMessageWithUser, DmChannel, FederationRelayProfileSnapshot, FederationIdentityDeleteS2SRequest, FederationProfileUpdatePayload, ServerEvent, ApprovalRequestSubscriberSummary, PeeringTriggerReason } from '@backspace/shared'; import { GROUP_DM_NAME_MIN_LENGTH, GROUP_DM_NAME_MAX_LENGTH } from '@backspace/shared/src/constants.js'; @@ -1120,6 +1121,16 @@ export async function federationRoutes(app: FastifyInstance): Promise { // Legitimate recovery path: local admin clicks "Reset peering" → // row is deleted → remote's /peer/accept then lands on a // non-existent row and the normal handshake path runs. + // + // Detection-only: if the inbound epoch differs from our trusted + // baseline, the peer is a NEW incarnation on the same domain (a + // wipe-and-reinstall). Route it to needs_attention + snapshot + + // journal — but STILL return 200 and STILL do not rekey. The + // anti-hijack guard above is preserved verbatim; detection never + // grants capability. + if (reqInstanceId && existing.peerInstanceId && reqInstanceId !== existing.peerInstanceId) { + markPeerReset(existing.id, sourceOrigin, existing.peerInstanceId, reqInstanceId); + } return reply.code(200).send({ accepted: true, instanceName: ourInstanceName, instanceId: ourInstanceId }); } if (existing.status === 'revoked') { @@ -1619,10 +1630,16 @@ export async function federationRoutes(app: FastifyInstance): Promise { }); } - const reachable = await probePeerReachable(peer.origin); + const probe = await probePeerReachable(peer.origin); - if (reachable) { - await markPeerRecovered(peer.id); + if (probe.reachable) { + const outcome = await recoverOrDetectReset(peer, probe); + if (outcome === 'reset_detected') { + // The peer is a new incarnation on the same domain. It was routed to + // needs_attention (detection-only, no rekey) and must NOT be recovered + // to active until an admin re-peers through the authenticated path. + return reply.code(200).send({ recovered: false, status: 'needs_attention' }); + } return reply.code(200).send({ recovered: true, status: 'active' }); } diff --git a/packages/server/src/utils/federationRecovery.test.ts b/packages/server/src/utils/federationRecovery.test.ts index fea143cd..db91d1e1 100644 --- a/packages/server/src/utils/federationRecovery.test.ts +++ b/packages/server/src/utils/federationRecovery.test.ts @@ -17,6 +17,11 @@ vi.mock('../db/index.js', () => ({ getDb: () => testDb, schema })); const onPeerActivated = vi.fn(); vi.mock('./federationPeerActivation.js', () => ({ onPeerActivated })); +const sendToAdmins = vi.fn(); +vi.mock('../ws/handler.js', () => ({ + connectionManager: { sendToAdmins, getAllOnlineUserIds: () => [], sendToUser: vi.fn() }, +})); + function applyMigrations(db: Database.Database): void { const dir = path.resolve(__dirname, '../../drizzle'); for (const f of fs.readdirSync(dir).filter(x => x.endsWith('.sql')).sort()) { @@ -45,23 +50,35 @@ describe('federationRecovery primitives', () => { vi.clearAllMocks(); }); - it('probePeerReachable returns true on a 200 from /api/instance/info', async () => { - const spy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('{}', { status: 200 })); + it('probePeerReachable returns reachable + parsed instanceId on a 200 from /api/instance/info', async () => { + const spy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('{"instanceId":"epoch-x"}', { status: 200 })); const { probePeerReachable } = await import('./federationRecovery.js'); - await expect(probePeerReachable('https://peer.example')).resolves.toBe(true); + await expect(probePeerReachable('https://peer.example')).resolves.toEqual({ reachable: true, instanceId: 'epoch-x' }); expect(spy).toHaveBeenCalledWith('https://peer.example/api/instance/info', expect.anything()); }); - it('probePeerReachable returns false on a non-ok response', async () => { - vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('', { status: 502 })); + it('probePeerReachable reports null instanceId when the body omits it (legacy peer)', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('{}', { status: 200 })); const { probePeerReachable } = await import('./federationRecovery.js'); - await expect(probePeerReachable('https://peer.example')).resolves.toBe(false); + await expect(probePeerReachable('https://peer.example')).resolves.toEqual({ reachable: true, instanceId: null }); }); - it('probePeerReachable returns false on network error', async () => { + it('probePeerReachable reports null instanceId when the body is unparseable', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('not-json', { status: 200 })); + const { probePeerReachable } = await import('./federationRecovery.js'); + await expect(probePeerReachable('https://peer.example')).resolves.toEqual({ reachable: true, instanceId: null }); + }); + + it('probePeerReachable returns not-reachable on a non-ok response', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('', { status: 502 })); + const { probePeerReachable } = await import('./federationRecovery.js'); + await expect(probePeerReachable('https://peer.example')).resolves.toEqual({ reachable: false, instanceId: null }); + }); + + it('probePeerReachable returns not-reachable on network error', async () => { vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('ENOTFOUND')); const { probePeerReachable } = await import('./federationRecovery.js'); - await expect(probePeerReachable('https://peer.example')).resolves.toBe(false); + await expect(probePeerReachable('https://peer.example')).resolves.toEqual({ reachable: false, instanceId: null }); }); it('markPeerRecovered flips status to active, resets pacing + counters, calls onPeerActivated', async () => { @@ -77,4 +94,60 @@ describe('federationRecovery primitives', () => { expect(row.lastSeenAt).toBeGreaterThan(0); expect(onPeerActivated).toHaveBeenCalledWith('peer-rec', 'health_check_recovery'); }); + + it('recoverOrDetectReset recovers when the probed epoch matches the trusted baseline', async () => { + testDb.insert(schema.federationPeers).values({ + id: 'peer-match', origin: 'https://peer.example', hmacSecret: 'secret', + status: 'unreachable', consecutiveFailures: 5, probeAttempts: 2, lastProbeAt: Date.now(), + peerInstanceId: 'E0', lastSyncedAt: Date.now(), createdAt: Date.now(), + }).run(); + const { recoverOrDetectReset } = await import('./federationRecovery.js'); + const outcome = await recoverOrDetectReset( + { id: 'peer-match', origin: 'https://peer.example', peerInstanceId: 'E0' }, + { reachable: true, instanceId: 'E0' }, + ); + expect(outcome).toBe('recovered'); + const row = testDb.select().from(schema.federationPeers) + .where(eq(schema.federationPeers.id, 'peer-match')).get()!; + expect(row.status).toBe('active'); + expect(onPeerActivated).toHaveBeenCalledWith('peer-match', 'health_check_recovery'); + }); + + it('recoverOrDetectReset recovers when the baseline is null (never-tracked / legacy)', async () => { + testDb.insert(schema.federationPeers).values({ + id: 'peer-null', origin: 'https://peer.example', hmacSecret: 'secret', + status: 'unreachable', consecutiveFailures: 5, probeAttempts: 2, lastProbeAt: Date.now(), + peerInstanceId: null, lastSyncedAt: Date.now(), createdAt: Date.now(), + }).run(); + const { recoverOrDetectReset } = await import('./federationRecovery.js'); + const outcome = await recoverOrDetectReset( + { id: 'peer-null', origin: 'https://peer.example', peerInstanceId: null }, + { reachable: true, instanceId: 'E9' }, + ); + expect(outcome).toBe('recovered'); + expect(testDb.select().from(schema.federationPeers) + .where(eq(schema.federationPeers.id, 'peer-null')).get()!.status).toBe('active'); + }); + + it('recoverOrDetectReset routes to needs_attention (does NOT recover) when the probed epoch differs', async () => { + testDb.insert(schema.federationPeers).values({ + id: 'peer-reset', origin: 'https://peer.example', hmacSecret: 'secret', + status: 'unreachable', consecutiveFailures: 5, probeAttempts: 2, lastProbeAt: Date.now(), + peerInstanceId: 'E0', lastSyncedAt: Date.now(), createdAt: Date.now(), + }).run(); + const { recoverOrDetectReset } = await import('./federationRecovery.js'); + const outcome = await recoverOrDetectReset( + { id: 'peer-reset', origin: 'https://peer.example', peerInstanceId: 'E0' }, + { reachable: true, instanceId: 'E1' }, + ); + expect(outcome).toBe('reset_detected'); + const row = testDb.select().from(schema.federationPeers) + .where(eq(schema.federationPeers.id, 'peer-reset')).get()!; + expect(row.status).toBe('needs_attention'); + expect(row.needsAttentionReason).toBe('peer_reset_detected'); + expect(row.peerInstanceId).toBe('E0'); // trusted baseline untouched + expect(row.hmacSecret).toBe('secret'); // never rekeyed + // A reset peer must NOT be recovered to active. + expect(onPeerActivated).not.toHaveBeenCalled(); + }); }); diff --git a/packages/server/src/utils/federationRecovery.ts b/packages/server/src/utils/federationRecovery.ts index 1bfffef7..c9734173 100644 --- a/packages/server/src/utils/federationRecovery.ts +++ b/packages/server/src/utils/federationRecovery.ts @@ -2,25 +2,55 @@ import { getDb } from '../db/index.js'; import * as schema from '../db/schema.js'; import { eq } from 'drizzle-orm'; import { onPeerActivated } from './federationPeerActivation.js'; +import { markPeerReset } from './federationReset.js'; /** Reachability-probe timeout (ms). */ export const RECOVERY_PROBE_TIMEOUT_MS = 10_000; +/** + * Result of a reachability probe. `instanceId` is the peer's advertised instance + * epoch (from `/api/instance/info`), used for reset detection. It is `null` when + * the peer is unreachable, when it is too old to advertise an epoch, or when the + * body is unparseable — all of which degrade to "no reset observed." + */ +export interface ProbeResult { + reachable: boolean; + instanceId: string | null; +} + /** * Liveness probe shared by the recovery tick and the manual recheck endpoint. * GET {origin}/api/instance/info with a 10s timeout. No HMAC — reachability is * not trust; a recovered-but-HMAC-broken peer still transitions to * needs_attention via the auth-failure path on the next real delivery. + * + * Also parses the peer's advertised `instanceId` (instance epoch) from the + * response so callers can detect a wipe-and-reinstall (a NEW incarnation on the + * same domain). A missing/unparseable epoch is reported as `null` — never an + * error — so a legacy peer that omits it simply recovers normally. */ -export async function probePeerReachable(origin: string, signal?: AbortSignal): Promise { +export async function probePeerReachable(origin: string, signal?: AbortSignal): Promise { try { const timeout = AbortSignal.timeout(RECOVERY_PROBE_TIMEOUT_MS); const response = await fetch(`${origin}/api/instance/info`, { signal: signal ? AbortSignal.any([signal, timeout]) : timeout, }); - return response.ok; + if (!response.ok) { + return { reachable: false, instanceId: null }; + } + let instanceId: string | null = null; + try { + const body = (await response.json()) as { instanceId?: unknown }; + if (typeof body?.instanceId === 'string' && body.instanceId.length > 0) { + instanceId = body.instanceId; + } + } catch { + // Reachable but body unparseable — treat epoch as unknown, not a failure. + instanceId = null; + } + return { reachable: true, instanceId }; } catch { - return false; + return { reachable: false, instanceId: null }; } } @@ -43,3 +73,33 @@ export async function markPeerRecovered(peerId: string): Promise { .run(); await onPeerActivated(peerId, 'health_check_recovery'); } + +/** + * Decide the outcome of a successful reachability probe for a peer that is + * eligible to recover. This is the single recovery-decision point shared by the + * background recovery worker and the manual recheck endpoint. + * + * Detection-only reset gate: if the peer has a trusted baseline epoch + * (`peer_instance_id`) AND the probe observed a DIFFERENT epoch, the peer is a + * new incarnation on the same domain. A genuinely reset peer's HMAC secret is + * desynced, so flipping it back to `active` via a reachability probe would + * resume relay against a dead secret. We therefore route it to + * `needs_attention` via `markPeerReset` and DO NOT recover it — it must wait for + * an admin-authenticated re-handshake. Only when the epoch matches the baseline + * (or the baseline is null / the epoch is unknown) does the normal recovery path + * run. + * + * @returns `'reset_detected'` if the peer was routed to needs_attention; + * `'recovered'` if it was flipped back to active. + */ +export async function recoverOrDetectReset( + peer: { id: string; origin: string; peerInstanceId: string | null }, + result: ProbeResult, +): Promise<'recovered' | 'reset_detected'> { + if (peer.peerInstanceId && result.instanceId && result.instanceId !== peer.peerInstanceId) { + markPeerReset(peer.id, peer.origin, peer.peerInstanceId, result.instanceId); + return 'reset_detected'; + } + await markPeerRecovered(peer.id); + return 'recovered'; +} diff --git a/packages/server/src/utils/federationReset.test.ts b/packages/server/src/utils/federationReset.test.ts new file mode 100644 index 00000000..b0396668 --- /dev/null +++ b/packages/server/src/utils/federationReset.test.ts @@ -0,0 +1,163 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import Database from 'better-sqlite3'; +import { drizzle } from 'drizzle-orm/better-sqlite3'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import * as schema from '../db/schema.js'; +import { eq } from 'drizzle-orm'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +type TestDb = ReturnType>; +let sqlite: Database.Database; +let testDb: TestDb; + +vi.mock('../db/index.js', () => ({ getDb: () => testDb, getRawDb: () => sqlite, schema })); + +const sendToAdmins = vi.fn(); +vi.mock('../ws/handler.js', () => ({ + connectionManager: { sendToAdmins, getAllOnlineUserIds: () => [], sendToUser: vi.fn() }, +})); + +const STUB = '!federation-replicated'; +const ORIGIN = 'https://peer.example'; +const DOMAIN = 'peer.example'; + +function applyMigrations(db: Database.Database): void { + const dir = path.resolve(__dirname, '../../drizzle'); + for (const f of fs.readdirSync(dir).filter(x => x.endsWith('.sql')).sort()) { + const sqlText = fs.readFileSync(path.join(dir, f), 'utf8'); + for (const stmt of sqlText.split(/-->\s*statement-breakpoint/)) { + const clean = stmt.trim(); + if (clean) db.exec(clean); + } + } +} + +function seedPeer(): void { + testDb.insert(schema.federationPeers).values({ + id: 'peer-1', origin: ORIGIN, hmacSecret: 'trusted-secret', + status: 'active', peerInstanceId: 'E0', lastSeenAt: Date.now(), + lastSyncedAt: Date.now(), createdAt: Date.now(), + }).run(); +} + +function seedUser(id: string, opts: { passwordHash: string; isDeleted?: number; homeInstance?: string }): void { + testDb.insert(schema.users).values({ + id, username: `${id}@${DOMAIN}`, passwordHash: opts.passwordHash, + homeInstance: opts.homeInstance ?? DOMAIN, homeUserId: id, + isDeleted: opts.isDeleted ?? 0, createdAt: Date.now(), + }).run(); +} + +describe('markPeerReset — detection-only reset routing', () => { + beforeEach(() => { + sqlite = new Database(':memory:'); + testDb = drizzle(sqlite, { schema }); + applyMigrations(sqlite); + vi.clearAllMocks(); + }); + + afterEach(() => { + sqlite.close(); + }); + + it('routes peer to needs_attention, snapshots the dead incarnation, and journals the dead epoch', async () => { + seedPeer(); + // Two non-deleted users for the reset origin: one pure S2S stub, one real account. + seedUser('stub-1', { passwordHash: STUB }); + seedUser('real-1', { passwordHash: '$2b$10$realbcrypthash' }); + // A deleted stub for the same origin — must NOT be flagged. + seedUser('stub-deleted', { passwordHash: STUB, isDeleted: 1 }); + // An unrelated user on a different origin — must NOT be flagged. + seedUser('other-1', { passwordHash: STUB, homeInstance: 'elsewhere.example' }); + + const { markPeerReset } = await import('./federationReset.js'); + markPeerReset('peer-1', ORIGIN, 'E0', 'E1'); + + const peer = testDb.select().from(schema.federationPeers) + .where(eq(schema.federationPeers.id, 'peer-1')).get()!; + expect(peer.status).toBe('needs_attention'); + expect(peer.needsAttentionReason).toBe('peer_reset_detected'); + expect(peer.observedPeerInstanceId).toBe('E1'); + // Trusted baseline + secret are NEVER touched by detection. + expect(peer.peerInstanceId).toBe('E0'); + expect(peer.hmacSecret).toBe('trusted-secret'); + + const flag = (id: string) => testDb.select().from(schema.users) + .where(eq(schema.users.id, id)).get()!.federationHealPending; + expect(flag('stub-1')).toBe(1); + expect(flag('real-1')).toBe(1); + expect(flag('stub-deleted')).toBe(0); // deleted → excluded from snapshot + expect(flag('other-1')).toBe(0); // different origin → excluded + + const journal = testDb.select().from(schema.federationResetEvents) + .where(eq(schema.federationResetEvents.origin, ORIGIN)).get()!; + expect(journal.deadEpoch).toBe('E0'); + expect(journal.newEpoch).toBeNull(); + expect(journal.resolvedAt).toBeNull(); + expect(journal.stubCount).toBe(1); // stub-1 only (deleted stub excluded) + expect(journal.orphanedAccountCount).toBe(1); // real-1 + + // Admin broadcast fired. + expect(sendToAdmins).toHaveBeenCalledWith({ type: 'federation_peers_changed' }); + expect(sendToAdmins).toHaveBeenCalledWith({ type: 'federation_peer_reset_detected', origin: ORIGIN }); + }); + + it('double-reset keeps the original dead_epoch and detected_at (only counts refresh)', async () => { + seedPeer(); + seedUser('stub-1', { passwordHash: STUB }); + seedUser('real-1', { passwordHash: '$2b$10$realbcrypthash' }); + + const { markPeerReset } = await import('./federationReset.js'); + markPeerReset('peer-1', ORIGIN, 'E0', 'E1'); + const first = testDb.select().from(schema.federationResetEvents) + .where(eq(schema.federationResetEvents.origin, ORIGIN)).get()!; + const originalDetectedAt = first.detectedAt; + + // Peer resets AGAIN before an admin resolved the first reset. + markPeerReset('peer-1', ORIGIN, 'E0', 'E2'); + const second = testDb.select().from(schema.federationResetEvents) + .where(eq(schema.federationResetEvents.origin, ORIGIN)).get()!; + + // The dead epoch is the ALREADY-snapshotted incarnation — never overwritten. + expect(second.deadEpoch).toBe('E0'); + expect(second.detectedAt).toBe(originalDetectedAt); + expect(second.resolvedAt).toBeNull(); + // The observed epoch on the peer row does advance to the newest observation. + expect(testDb.select().from(schema.federationPeers) + .where(eq(schema.federationPeers.id, 'peer-1')).get()!.observedPeerInstanceId).toBe('E2'); + }); + + it('a resolved prior reset starts a fresh journal entry on a new reset', async () => { + seedPeer(); + seedUser('stub-1', { passwordHash: STUB }); + + const { markPeerReset } = await import('./federationReset.js'); + markPeerReset('peer-1', ORIGIN, 'E0', 'E1'); + // Simulate the heal having resolved the first reset. + testDb.update(schema.federationResetEvents) + .set({ resolvedAt: Date.now(), newEpoch: 'E1' }) + .where(eq(schema.federationResetEvents.origin, ORIGIN)).run(); + + // A brand-new reset lands: dead_epoch should update to the new baseline. + markPeerReset('peer-1', ORIGIN, 'E1', 'E2'); + const row = testDb.select().from(schema.federationResetEvents) + .where(eq(schema.federationResetEvents.origin, ORIGIN)).get()!; + expect(row.deadEpoch).toBe('E1'); + expect(row.newEpoch).toBeNull(); + expect(row.resolvedAt).toBeNull(); + }); + + it('matches home_instance stored as a full URL (defensive format match)', async () => { + seedPeer(); + // Legacy straggler stored with the https:// prefix rather than bare domain. + seedUser('stub-url', { passwordHash: STUB, homeInstance: ORIGIN }); + + const { markPeerReset } = await import('./federationReset.js'); + markPeerReset('peer-1', ORIGIN, 'E0', 'E1'); + + expect(testDb.select().from(schema.users) + .where(eq(schema.users.id, 'stub-url')).get()!.federationHealPending).toBe(1); + }); +}); diff --git a/packages/server/src/utils/federationReset.ts b/packages/server/src/utils/federationReset.ts new file mode 100644 index 00000000..f05fa706 --- /dev/null +++ b/packages/server/src/utils/federationReset.ts @@ -0,0 +1,148 @@ +import { and, eq, sql } from 'drizzle-orm'; +import { getDb, schema } from '../db/index.js'; +import { extractDomain } from '../routes/federation.js'; +import { connectionManager } from '../ws/handler.js'; + +/** Pure-stub sentinel: a user replicated purely over S2S (no local credentials). */ +const REPLICATED_STUB_SENTINEL = '!federation-replicated'; + +/** + * SQL predicate matching every local user whose home instance is `origin`. + * + * `users.home_instance` is stored canonically as a bare domain + * (`resolveOrCreateReplicatedUser` writes `extractDomain(...)`), so we key on + * the bare domain. We additionally match the `https://`/`http://`-prefixed + * forms so any legacy full-URL straggler is still caught — mirroring the + * defensive normalization used across the outbox/worker paths. A silent + * zero-match here would no-op the entire heal, so the match is deliberately + * permissive on format while exact on domain. + */ +export function homeInstanceMatch(origin: string) { + const domain = extractDomain(origin); + return sql`(${schema.users.homeInstance} = ${domain} OR ${schema.users.homeInstance} = ${'https://' + domain} OR ${schema.users.homeInstance} = ${'http://' + domain})`; +} + +/** + * Detection-only reset routing. Invoked when a peer behind a known origin is + * observed to carry a DIFFERENT instance epoch than the trusted baseline + * (`federation_peers.peer_instance_id`) — i.e. the instance was wiped and a new + * incarnation stood up on the same domain. + * + * This routes the peer to `needs_attention` (reason `peer_reset_detected`), + * snapshots the dead incarnation's users (`federation_heal_pending = 1`), and + * journals the dead epoch durably in `federation_reset_events`. It then notifies + * admins. + * + * It performs **NO rekey, NO tombstone, NO handle change, NO content deletion**. + * The trusted baseline (`peer_instance_id`) and the `hmac_secret` are left + * untouched — the observed (but not yet trusted) epoch is recorded separately in + * `observed_peer_instance_id`. Trust re-establishment is admin-gated (§5) and + * the actual data heal fires only after an authenticated re-peer (§6). Because + * none of this grants capability or destroys content, it is safe to fire on an + * unauthenticated detection signal: the worst a spoofed detection can do is flag + * a peer for admin review. + * + * Idempotent: if an UNRESOLVED reset row already exists for the origin (the peer + * reset again before an admin resolved the first), the original `dead_epoch` and + * `detected_at` are preserved — that is the incarnation whose users are already + * snapshotted — and only the summary counts are refreshed. + * + * @param peerId `federation_peers.id` of the reset peer. + * @param origin The peer origin (bare domain or full URL). + * @param deadEpoch The peer's trusted baseline epoch at detection time. + * @param observedEpoch The new epoch observed on the peer. + */ +export function markPeerReset(peerId: string, origin: string, deadEpoch: string, observedEpoch: string): void { + const db = getDb(); + + db.transaction((tx) => { + // 1. Route the peer to needs_attention and record the observed (untrusted) + // epoch. peer_instance_id (trusted baseline) and hmac_secret are NOT + // touched — an unauthenticated observation never rekeys trust. + tx.update(schema.federationPeers) + .set({ + status: 'needs_attention', + needsAttentionReason: 'peer_reset_detected', + observedPeerInstanceId: observedEpoch, + }) + .where(eq(schema.federationPeers.id, peerId)) + .run(); + + // 2. Snapshot exactly the current (dead-incarnation) users for this origin. + // Any stub created AFTER this point (e.g. a friend-add reaching the new + // incarnation directly) is un-flagged and survives the heal. + tx.update(schema.users) + .set({ federationHealPending: 1 }) + .where(and(eq(schema.users.isDeleted, 0), homeInstanceMatch(origin))) + .run(); + + // 3. Compute summary counts for the admin surface, over the freshly-flagged + // set: pure replicated stubs vs. real federated accounts (local content). + const stubCount = tx + .select({ n: sql`count(*)` }) + .from(schema.users) + .where(and( + eq(schema.users.federationHealPending, 1), + eq(schema.users.passwordHash, REPLICATED_STUB_SENTINEL), + homeInstanceMatch(origin), + )) + .get()?.n ?? 0; + + const orphanedAccountCount = tx + .select({ n: sql`count(*)` }) + .from(schema.users) + .where(and( + eq(schema.users.federationHealPending, 1), + sql`${schema.users.passwordHash} != ${REPLICATED_STUB_SENTINEL}`, + homeInstanceMatch(origin), + )) + .get()?.n ?? 0; + + // 4. Journal the dead incarnation durably. This row survives the peer-row + // deletion that Re-peer performs, preserving dead_epoch for the + // false-positive guard and the admin surface. + const existing = tx + .select() + .from(schema.federationResetEvents) + .where(eq(schema.federationResetEvents.origin, origin)) + .get(); + + if (existing && existing.resolvedAt === null) { + // Double-reset: keep the ORIGINAL dead_epoch + detected_at (the + // incarnation already snapshotted), refresh counts only. Never overwrite + // dead_epoch on an unresolved row. + tx.update(schema.federationResetEvents) + .set({ stubCount, orphanedAccountCount }) + .where(eq(schema.federationResetEvents.origin, origin)) + .run(); + } else { + // First detection for this origin, or a prior reset that was already + // resolved — start a fresh journal entry. + tx.insert(schema.federationResetEvents) + .values({ + origin, + deadEpoch, + newEpoch: null, + detectedAt: Date.now(), + resolvedAt: null, + stubCount, + orphanedAccountCount, + }) + .onConflictDoUpdate({ + target: schema.federationResetEvents.origin, + set: { + deadEpoch, + newEpoch: null, + detectedAt: Date.now(), + resolvedAt: null, + stubCount, + orphanedAccountCount, + }, + }) + .run(); + } + }); + + connectionManager.sendToAdmins({ type: 'federation_peers_changed' as const }); + connectionManager.sendToAdmins({ type: 'federation_peer_reset_detected' as const, origin }); +} diff --git a/packages/server/src/utils/federationWorker.ts b/packages/server/src/utils/federationWorker.ts index 7d6ffc8d..ecfbd02b 100644 --- a/packages/server/src/utils/federationWorker.ts +++ b/packages/server/src/utils/federationWorker.ts @@ -12,7 +12,7 @@ import { connectionManager } from '../ws/handler.js'; import { generateThumbnail } from './thumbnail.js'; import type { FederationRelayRequest, FederationRelayResponse, FederationRelayEvent } from '@backspace/shared'; import { startupBootstrapSync, onPeerDeactivated } from './federationPeerActivation.js'; -import { probePeerReachable, markPeerRecovered } from './federationRecovery.js'; +import { probePeerReachable, recoverOrDetectReset } from './federationRecovery.js'; import { backfillReplicatedProfileAssets } from '../routes/federation.js'; import { invokePermanentFailureCallback } from './federationRollback.js'; import { refreshPeerEpochs, getInstanceId } from './federationEpoch.js'; @@ -1057,11 +1057,15 @@ export async function processRecoveryTick(): Promise { if (!due) continue; recoveryAbortController = new AbortController(); - const reachable = await probePeerReachable(peer.origin, recoveryAbortController.signal); + const probe = await probePeerReachable(peer.origin, recoveryAbortController.signal); - if (reachable) { - await markPeerRecovered(peer.id); - console.log(`[federation-worker] Peer ${peer.origin} recovered — marked active`); + if (probe.reachable) { + const outcome = await recoverOrDetectReset(peer, probe); + if (outcome === 'reset_detected') { + console.warn(`[federation-worker] Peer ${peer.origin} reset detected (new instance epoch) — routed to needs_attention`); + } else { + console.log(`[federation-worker] Peer ${peer.origin} recovered — marked active`); + } } else { db.update(schema.federationPeers) .set({ probeAttempts: peer.probeAttempts + 1, lastProbeAt: now }) diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index aa2796a7..066157ea 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -486,6 +486,7 @@ export type ServerEvent = | { type: 'federation_peer_rejected'; peerOrigin: string; peerLabel?: string; reason: string; affectedContexts: Array<{ contextType: 'dm' | 'friend'; contextId: string; contextLabel: string }> } | { type: 'federation_peer_active'; peerOrigin: string } | { type: 'federation_peers_changed' } + | { type: 'federation_peer_reset_detected'; origin: string } | { type: 'federation_approval_request_received'; origin: string; instanceName?: string } | { type: 'peering_subscription_changed' } | { type: 'peering_notification_received'; kind: PeeringNotificationKind } From 8ae8dcfd86092c09eefa64289bb599d3d06d25a2 Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Wed, 1 Jul 2026 22:17:14 +0200 Subject: [PATCH 08/12] feat(federation): heal on re-peer with false-positive guard Add healResetIncarnation (federationReset.ts): fires from onPeerActivated after an authenticated re-peer to soft-tombstone the flagged pure S2S stubs of a reset peer's dead incarnation, clearing stale friendships/DMs so the reported bug is fixed. Two mandatory guards: a reason gate (allow-list of 8 genuine handshake activation reasons; excludes health_check_recovery + startup_bootstrap so their stale baseline can never silently resolve a journal without healing) and an epoch comparison (dead_epoch === newEpoch => false alarm, no tombstone). Uses tombstoneUser(uid, { purgeContent: false }); real federated accounts are left flagged + intact for Phase 2. Runs outside any transaction. Wire into onPeerActivated before the mutation-log re-sync. --- docs/systems/federation.md | 16 ++ .../src/utils/federationPeerActivation.ts | 20 +++ .../server/src/utils/federationReset.test.ts | 116 +++++++++++++++ packages/server/src/utils/federationReset.ts | 139 +++++++++++++++++- 4 files changed, 290 insertions(+), 1 deletion(-) diff --git a/docs/systems/federation.md b/docs/systems/federation.md index d44b5ab9..69f8f1c2 100644 --- a/docs/systems/federation.md +++ b/docs/systems/federation.md @@ -333,6 +333,22 @@ In a single transaction, `markPeerReset`: Legacy peers advertise no epoch (`instanceId` null), so detection requires a non-null observed epoch differing from a non-null stored baseline — legacy peers never trigger it, and the existing `auth_failures → needs_attention → manual Reset` path continues unchanged for them. +### Data Self-Heal (`healResetIncarnation` — `utils/federationReset.ts`) + +Detection (`markPeerReset`) only snapshots + journals + notifies; it destroys nothing. The actual heal is `healResetIncarnation(origin, newEpoch, reason)`, fired from `onPeerActivated` (`utils/federationPeerActivation.ts`) **after an admin-authenticated re-peer**, keyed to the confirmed epoch change (design §6). It runs **before** the mutation-log re-sync in `onPeerActivated` so re-sync repopulates onto a clean slate, and it runs **outside any transaction** (`tombstoneUser` opens its own; better-sqlite3 throws on a nested `BEGIN`). + +**Two mandatory guards, in order:** + +1. **Reason gate.** `onPeerActivated` fires on non-handshake paths too. Only genuine re-handshake reasons carry a freshly-exchanged, trustworthy epoch. `healResetIncarnation` returns immediately unless `reason` is in the allow-list `HANDSHAKE_ACTIVATION_REASONS` (typed `ReadonlySet`): `initiate_accepted`, `accept_new`, `accept_pending`, `accept_rejected_override`, `accept_awaiting_approval`, `accept_awaiting_approval_fallback`, `approval_handshake`, `ensure_peered`. The two EXCLUDED reasons — `health_check_recovery` (reachability flip in `markPeerRecovered`) and `startup_bootstrap` (boot re-scan) — flip a peer to `active` **without** a handshake, so their baseline is stale (still equals the journaled `dead_epoch`). Without the gate they would hit the `deadEpoch === newEpoch` false-alarm branch and silently resolve the journal + clear the flags WITHOUT healing, permanently burying the bug. Gated out, they leave the journal fully intact for a later genuine re-handshake to heal. + +2. **Epoch comparison (false-positive guard).** For a gated-in reason, look up the UNRESOLVED `federation_reset_events` row for the origin (none → return): + - **`journal.dead_epoch === newEpoch`** — the re-peer confirmed the SAME incarnation (spurious/spoofed detection, or an admin re-peer to a never-reset live peer). **NO tombstone** — the user-level snapshot flags alone must never authorize destruction; only a confirmed epoch change does. Clears `federation_heal_pending` for the origin and resolves the journal (`new_epoch`, `resolved_at`). + - **`journal.dead_epoch !== newEpoch`** — a GENUINE new incarnation. Soft-tombstones the flagged **pure stubs** only, then clears their flags and resolves the journal. + +**Soft-tombstone (pure stubs only).** For every user that is `federation_heal_pending = 1` AND `password_hash = '!federation-replicated'` (pure S2S stub sentinel) AND matches the origin (`homeInstanceMatch`), calls `tombstoneUser(uid, { purgeContent: false })`. The `purgeContent: false` is **non-negotiable** — the default (`true`) irreversibly deletes this box's reactions and authored space messages, violating the invariant that a remote's reset never destroys our non-re-syncable content. The soft tombstone clears exactly the relationship rows that cause the bug (`friends`, `friend_requests`, `dm_members`, …) so stale friendships/DMs clear and re-adds work. Flags are then cleared **keyed by the stub id list** (not by re-querying the sentinel — `tombstoneUser` has already randomized `password_hash`). + +**Real federated accounts left intact.** A flagged user that is NOT a stub (`password_hash != '!federation-replicated'`) carries real, non-re-syncable local content. It is **never** auto-tombstoned — it stays `federation_heal_pending = 1` and fully intact for the Phase 2 quarantine/admin surface (design §6.3). + ### S2S Identity Deletion (`DELETE /api/federation/identity`) Allows a home instance to remove a user's replicated identity from a remote instance. diff --git a/packages/server/src/utils/federationPeerActivation.ts b/packages/server/src/utils/federationPeerActivation.ts index da131d12..07b767d3 100644 --- a/packages/server/src/utils/federationPeerActivation.ts +++ b/packages/server/src/utils/federationPeerActivation.ts @@ -4,6 +4,7 @@ import { and, eq } from 'drizzle-orm'; import { isFederationRelayEnabled } from './federationOutbox.js'; import { buildFederationHeaders, getOurOrigin } from './federationAuth.js'; import { generateSnowflake } from './snowflake.js'; +import { healResetIncarnation } from './federationReset.js'; import type { FederationRelayEvent } from '@backspace/shared'; export type PeerActivationReason = @@ -50,6 +51,25 @@ export async function onPeerActivated( const promise = (async () => { try { resetOutboxBackoff(peerId); + + // Instance-epoch self-heal. If this origin has an unresolved reset journal + // AND this is a genuine re-handshake activation (reason gate lives inside + // healResetIncarnation), heal the dead incarnation's stale stubs BEFORE the + // mutation-log re-sync below — so re-sync repopulates onto a clean slate + // (design §6.1). By this point the activation path has already (re)written + // peer_instance_id to the freshly-exchanged epoch. Runs OUTSIDE any + // transaction: tombstoneUser opens its own, and better-sqlite3 throws on a + // nested BEGIN. No-op on non-handshake reasons (health_check_recovery / + // startup_bootstrap) and when no reset is journaled. + const resetPeerRow = getDb() + .select({ origin: schema.federationPeers.origin, epoch: schema.federationPeers.peerInstanceId }) + .from(schema.federationPeers) + .where(eq(schema.federationPeers.id, peerId)) + .get(); + if (resetPeerRow?.epoch) { + healResetIncarnation(resetPeerRow.origin, resetPeerRow.epoch, reason); + } + await syncPeerMutationLog(peerId, reason); await fanoutOutboundSubscribers(peerId); diff --git a/packages/server/src/utils/federationReset.test.ts b/packages/server/src/utils/federationReset.test.ts index b0396668..6d7a149a 100644 --- a/packages/server/src/utils/federationReset.test.ts +++ b/packages/server/src/utils/federationReset.test.ts @@ -161,3 +161,119 @@ describe('markPeerReset — detection-only reset routing', () => { .where(eq(schema.users.id, 'stub-url')).get()!.federationHealPending).toBe(1); }); }); + +describe('healResetIncarnation — heal after authenticated re-peer', () => { + beforeEach(() => { + sqlite = new Database(':memory:'); + testDb = drizzle(sqlite, { schema }); + applyMigrations(sqlite); + vi.clearAllMocks(); + }); + + afterEach(() => { + sqlite.close(); + }); + + function seedJournal(deadEpoch: string): void { + testDb.insert(schema.federationResetEvents).values({ + origin: ORIGIN, deadEpoch, newEpoch: null, + detectedAt: Date.now(), resolvedAt: null, + stubCount: 1, orphanedAccountCount: 1, + }).run(); + } + + function flag(id: string): void { + testDb.update(schema.users).set({ federationHealPending: 1 }) + .where(eq(schema.users.id, id)).run(); + } + + it('genuine reset: soft-tombstones flagged stubs only, leaves real accounts flagged + intact, resolves journal', async () => { + seedPeer(); + seedJournal('E0'); + // A local native user to be the friendship counterpart. + testDb.insert(schema.users).values({ + id: 'local-1', username: 'alice', passwordHash: '$2b$10$localhash', + homeInstance: null, homeUserId: null, isDeleted: 0, createdAt: Date.now(), + }).run(); + // Flagged pure S2S stub with a friendship to the local user. + seedUser('stub-1', { passwordHash: STUB }); + flag('stub-1'); + testDb.insert(schema.friends).values({ + userId: 'stub-1', friendId: 'local-1', createdAt: Date.now(), + }).run(); + // Flagged REAL federated account (real bcrypt) — must survive untouched + still flagged. + seedUser('real-1', { passwordHash: '$2b$10$realbcrypthash' }); + flag('real-1'); + + const { healResetIncarnation } = await import('./federationReset.js'); + healResetIncarnation(ORIGIN, 'E1', 'initiate_accepted'); + + // Stub soft-tombstoned. + const stub = testDb.select().from(schema.users).where(eq(schema.users.id, 'stub-1')).get()!; + expect(stub.isDeleted).toBe(1); + expect(stub.username).toBe('!deleted:stub-1'); + // Its friendship row is gone → re-adds work again. + expect(testDb.select().from(schema.friends) + .where(eq(schema.friends.userId, 'stub-1')).all()).toHaveLength(0); + // Heal flag cleared on the healed stub. + expect(stub.federationHealPending).toBe(0); + + // Real account UNTOUCHED and STILL flagged (left for Phase 2 quarantine). + const real = testDb.select().from(schema.users).where(eq(schema.users.id, 'real-1')).get()!; + expect(real.isDeleted).toBe(0); + expect(real.username).toBe('real-1@peer.example'); + expect(real.federationHealPending).toBe(1); + + // Journal resolved with the freshly-handshaked epoch. + const journal = testDb.select().from(schema.federationResetEvents) + .where(eq(schema.federationResetEvents.origin, ORIGIN)).get()!; + expect(journal.resolvedAt).not.toBeNull(); + expect(journal.newEpoch).toBe('E1'); + }); + + it('false positive (re-peer confirmed same incarnation): NO tombstone, flags cleared, journal resolved', async () => { + seedPeer(); + seedJournal('E0'); + seedUser('stub-1', { passwordHash: STUB }); + flag('stub-1'); + + const { healResetIncarnation } = await import('./federationReset.js'); + healResetIncarnation(ORIGIN, 'E0', 'accept_new'); // dead_epoch === newEpoch → false alarm + + const stub = testDb.select().from(schema.users).where(eq(schema.users.id, 'stub-1')).get()!; + expect(stub.isDeleted).toBe(0); // NOT tombstoned + expect(stub.username).toBe('stub-1@peer.example'); + expect(stub.federationHealPending).toBe(0); // flag cleared + + const journal = testDb.select().from(schema.federationResetEvents) + .where(eq(schema.federationResetEvents.origin, ORIGIN)).get()!; + expect(journal.resolvedAt).not.toBeNull(); + expect(journal.newEpoch).toBe('E0'); + }); + + it('recovery/startup flip must NOT resolve the journal or clear flags (the critical guard)', async () => { + seedPeer(); + seedJournal('E0'); + seedUser('stub-1', { passwordHash: STUB }); + flag('stub-1'); + + const { healResetIncarnation } = await import('./federationReset.js'); + + // Both non-handshake reasons flip a peer to active with a STALE baseline + // (still E0). Without the reason gate they'd hit dead_epoch === newEpoch and + // silently resolve the journal WITHOUT healing → the bug permanently buried. + for (const reason of ['health_check_recovery', 'startup_bootstrap'] as const) { + healResetIncarnation(ORIGIN, 'E0', reason); + + const journal = testDb.select().from(schema.federationResetEvents) + .where(eq(schema.federationResetEvents.origin, ORIGIN)).get()!; + expect(journal.resolvedAt, `reason=${reason}`).toBeNull(); + expect(journal.newEpoch, `reason=${reason}`).toBeNull(); + + const stub = testDb.select().from(schema.users) + .where(eq(schema.users.id, 'stub-1')).get()!; + expect(stub.federationHealPending, `reason=${reason}`).toBe(1); + expect(stub.isDeleted, `reason=${reason}`).toBe(0); + } + }); +}); diff --git a/packages/server/src/utils/federationReset.ts b/packages/server/src/utils/federationReset.ts index f05fa706..92eea755 100644 --- a/packages/server/src/utils/federationReset.ts +++ b/packages/server/src/utils/federationReset.ts @@ -1,7 +1,9 @@ -import { and, eq, sql } from 'drizzle-orm'; +import { and, eq, inArray, isNull, sql } from 'drizzle-orm'; import { getDb, schema } from '../db/index.js'; import { extractDomain } from '../routes/federation.js'; import { connectionManager } from '../ws/handler.js'; +import { tombstoneUser } from './userDeletion.js'; +import type { PeerActivationReason } from './federationPeerActivation.js'; /** Pure-stub sentinel: a user replicated purely over S2S (no local credentials). */ const REPLICATED_STUB_SENTINEL = '!federation-replicated'; @@ -146,3 +148,138 @@ export function markPeerReset(peerId: string, origin: string, deadEpoch: string, connectionManager.sendToAdmins({ type: 'federation_peers_changed' as const }); connectionManager.sendToAdmins({ type: 'federation_peer_reset_detected' as const, origin }); } + +/** + * Activation reasons that involve a fresh, HMAC-authenticated handshake which + * (re)writes `federation_peers.peer_instance_id`. ONLY these reasons carry a + * freshly-exchanged epoch that can be trusted to confirm-or-refute a reset. + * + * The two EXCLUDED members of `PeerActivationReason` — `health_check_recovery` + * (a reachability flip in `markPeerRecovered`) and `startup_bootstrap` (a boot + * re-scan) — flip a peer to `active` WITHOUT any handshake, so the baseline they + * observe is STALE (still equal to the journaled `dead_epoch`). Letting the heal + * run on those paths would take the `deadEpoch === newEpoch` false-alarm branch + * and silently resolve the reset journal + clear the snapshot flags WITHOUT ever + * healing — permanently burying the bug. The reason gate below stops that: on a + * non-handshake activation the journal is left fully intact for a later genuine + * re-handshake to heal. + * + * Typed as `ReadonlySet` so a typo or a future + * union-member rename is caught by tsc, not at runtime. + */ +const HANDSHAKE_ACTIVATION_REASONS: ReadonlySet = new Set([ + 'initiate_accepted', + 'accept_new', + 'accept_pending', + 'accept_rejected_override', + 'accept_awaiting_approval', + 'accept_awaiting_approval_fallback', + 'approval_handshake', + 'ensure_peered', +]); + +/** + * The data self-heal, fired from `onPeerActivated` AFTER an authenticated + * re-peer (design §6). It is the counterpart to `markPeerReset`'s detection: + * detection snapshots + journals but never destroys; this heals once — and only + * once — the epoch change has been proven through a genuine handshake. + * + * Two mandatory guards, in order: + * + * 1. **Reason gate.** Returns immediately unless `reason` is a genuine + * handshake activation (see `HANDSHAKE_ACTIVATION_REASONS`). Reachability / + * startup flips carry a stale baseline and must leave the journal untouched. + * + * 2. **Epoch comparison (false-positive guard).** For a gated-in reason, look up + * the UNRESOLVED `federation_reset_events` row for the origin. If none → no + * outstanding reset → return. + * - `journal.deadEpoch === newEpoch`: the re-peer confirmed the SAME + * incarnation (a spurious/spoofed detection, or an admin re-peer to the + * never-reset live peer). **No tombstone** — the user-level snapshot flags + * alone must never drive destruction; only a confirmed epoch change + * authorizes it. Clear all heal flags for the origin and resolve the + * journal. + * - `journal.deadEpoch !== newEpoch`: a GENUINE new incarnation. Soft- + * tombstone the flagged PURE STUBS only, then clear their flags and resolve + * the journal. + * + * Real federated accounts (`federation_heal_pending = 1` but NOT a stub) carry + * non-re-syncable local content and are **never** auto-tombstoned; they stay + * flagged + intact for the Phase 2 quarantine/admin surface (design §6.3). + * + * **Transaction hazard:** `tombstoneUser` opens its OWN `db.transaction`, and + * better-sqlite3 throws on a nested `BEGIN`. `healResetIncarnation` therefore + * runs its `select`/`update` calls UNWRAPPED (never inside a transaction) and + * calls `tombstoneUser` per-stub outside any open transaction. The caller + * (`onPeerActivated`) must likewise not invoke this from within a transaction. + * + * @param origin The reset peer's origin (bare domain or full URL). + * @param newEpoch The peer's freshly-handshaked epoch (`peer_instance_id`). + * @param reason The activation reason that triggered this call. + */ +export function healResetIncarnation(origin: string, newEpoch: string, reason: PeerActivationReason): void { + // Guard 1 — reason gate: only a genuine re-handshake carries a trustworthy epoch. + if (!HANDSHAKE_ACTIVATION_REASONS.has(reason)) return; + + const db = getDb(); + + const journal = db + .select() + .from(schema.federationResetEvents) + .where(and( + eq(schema.federationResetEvents.origin, origin), + isNull(schema.federationResetEvents.resolvedAt), + )) + .get(); + if (!journal) return; // no outstanding reset for this origin + + // Guard 2 — epoch comparison (the false-positive guard). + if (journal.deadEpoch === newEpoch) { + // FALSE ALARM: re-peer confirmed the SAME incarnation. The snapshot flags + // alone must NEVER drive a tombstone — clear them and resolve, no deletion. + db.update(schema.users) + .set({ federationHealPending: 0 }) + .where(and(eq(schema.users.federationHealPending, 1), homeInstanceMatch(origin))) + .run(); + db.update(schema.federationResetEvents) + .set({ newEpoch, resolvedAt: Date.now() }) + .where(eq(schema.federationResetEvents.origin, origin)) + .run(); + return; + } + + // GENUINE reset: soft-tombstone the flagged PURE STUBS only. + const stubs = db + .select({ id: schema.users.id }) + .from(schema.users) + .where(and( + eq(schema.users.federationHealPending, 1), + eq(schema.users.passwordHash, REPLICATED_STUB_SENTINEL), + homeInstanceMatch(origin), + )) + .all(); + + // MANDATORY: purgeContent:false — a soft tombstone. The default (true) would + // irreversibly delete this box's reactions / space messages, violating the + // §1 invariant that a remote's reset never destroys our non-re-syncable + // content. Each call opens its own transaction, so this loop stays UNWRAPPED. + for (const stub of stubs) { + tombstoneUser(stub.id, { purgeContent: false }); + } + + // Clear the heal flag on exactly the stubs we healed, keyed by id. + // `tombstoneUser` has already randomized their `password_hash`, so re-querying + // by the stub sentinel would miss them — the id list is the reliable key. + // Real accounts keep `federation_heal_pending = 1` for Phase 2. + if (stubs.length > 0) { + db.update(schema.users) + .set({ federationHealPending: 0 }) + .where(inArray(schema.users.id, stubs.map((s) => s.id))) + .run(); + } + + db.update(schema.federationResetEvents) + .set({ newEpoch, resolvedAt: Date.now() }) + .where(eq(schema.federationResetEvents.origin, origin)) + .run(); +} From 7ef1ded116783a0b3987fcde64ad04ef7e415f88 Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Wed, 1 Jul 2026 22:22:43 +0200 Subject: [PATCH 09/12] test(federation): lock reset admissibility for peer_reset_detected peers --- .../src/routes/federation.reset.test.ts | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/packages/server/src/routes/federation.reset.test.ts b/packages/server/src/routes/federation.reset.test.ts index d1d13aaa..aedd3b34 100644 --- a/packages/server/src/routes/federation.reset.test.ts +++ b/packages/server/src/routes/federation.reset.test.ts @@ -190,4 +190,49 @@ describe('POST /api/federation/peers/:id/reset', () => { const outbox = testDb.select().from(schema.federationOutbox).all(); expect(outbox).toHaveLength(0); }); + + it('admits a peer_reset_detected peer and cascade-removes its outbox', async () => { + // Detection routes a re-installed remote to needs_attention with the + // `peer_reset_detected` sub-reason. The one-click admin Re-peer action + // reuses this reset endpoint, so it must admit that peer exactly like an + // auth_failures one. The reset handler gates only on `status`, not on the + // reason — this locks that reason-agnostic guarantee in place. + const now = Date.now(); + testDb.insert(schema.federationPeers).values({ + id: 'peer-reset', + origin: 'https://reinstalled.example', + hmacSecret: 'b'.repeat(64), + status: 'needs_attention', + needsAttentionReason: 'peer_reset_detected', + createdAt: now, + }).run(); + + testDb.insert(schema.federationOutbox).values({ + id: 'out-reset-1', + peerId: 'peer-reset', + contextId: 'dm-1', + entityId: 'msg-1', + contextType: 'dm', + eventType: 'message_create', + payload: '{}', + nextRetryAt: now, + expiresAt: now + 86_400_000, + createdAt: now, + }).run(); + + const res = await app.inject({ + method: 'POST', + url: '/api/federation/peers/peer-reset/reset', + }); + expect(res.statusCode).toBe(200); + expect(res.json()).toEqual({ success: true }); + + // Peer row deleted + const peers = testDb.select().from(schema.federationPeers).all(); + expect(peers).toHaveLength(0); + + // Its outbox entries cascade-removed + const outbox = testDb.select().from(schema.federationOutbox).all(); + expect(outbox).toHaveLength(0); + }); }); From 7d8c9c9d8d8de64d3ca08079ba79bb9a47de7b01 Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Wed, 1 Jul 2026 22:32:40 +0200 Subject: [PATCH 10/12] feat(federation): peer_reset_pending guard during limbo window --- docs/systems/federation.md | 9 + docs/systems/social.md | 1 + .../src/routes/dm.peerResetPending.test.ts | 165 ++++++++++++++++++ packages/server/src/routes/dm.ts | 35 ++++ .../src/routes/social.federated.test.ts | 97 ++++++++++ packages/server/src/routes/social.ts | 28 ++- 6 files changed, 334 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/routes/dm.peerResetPending.test.ts diff --git a/docs/systems/federation.md b/docs/systems/federation.md index 69f8f1c2..eb87d518 100644 --- a/docs/systems/federation.md +++ b/docs/systems/federation.md @@ -333,6 +333,15 @@ In a single transaction, `markPeerReset`: Legacy peers advertise no epoch (`instanceId` null), so detection requires a non-null observed epoch differing from a non-null stored baseline — legacy peers never trigger it, and the existing `auth_failures → needs_attention → manual Reset` path continues unchanged for them. +### Limbo-window user error (`peer_reset_pending`) + +Between reset detection (`markPeerReset`) and the admin's one-click Re-peer, the stale identity graph still exists and no heal has run (design §5.3). During this window a user re-adding a same-name friend, or creating a DM to that origin, would otherwise hit a confusing `already_friends` (stale friendship bound to the dead incarnation) or `peer_rejected` (the peer now sits in `needs_attention`, tripping `ensurePeered`). Both user-facing hot paths short-circuit with a clearer **409 `{ error: 'peer_reset_pending' }`**: + +- **Friend-add** (`social.ts` `POST /api/social/requests`, federated branch): after `resolveOriginFromHostname` yields `peerOrigin` and **before** the peering/lookup/`already_friends` checks. +- **DM-create** (`dm.ts` `POST /api/dm`, `homeUserId + homeInstance` branch): before stub creation, so no un-flagged stub is left behind. The target origin is resolved with `resolveOriginFromHostname(new URL(canonicalizeHomeInstance(homeInstance)).host)`. + +Both perform an **O(1) point lookup** on the `federation_reset_events` origin PRIMARY KEY (`origin = peerOrigin AND resolved_at IS NULL`). `peerOrigin` (from `resolveOriginFromHostname`, which returns the stored `federation_peers.origin` verbatim) is exactly the string `markPeerReset` journals, so the query is a single indexed hit/miss. The guard **only** short-circuits when an unresolved row exists; the common case — no reset in progress — is one indexed miss and the normal path proceeds byte-for-byte unchanged. Once the admin re-peers and `healResetIncarnation` resolves the journal (`resolved_at` set), the guard stops firing and the freshly-clean graph accepts the re-add. + ### Data Self-Heal (`healResetIncarnation` — `utils/federationReset.ts`) Detection (`markPeerReset`) only snapshots + journals + notifies; it destroys nothing. The actual heal is `healResetIncarnation(origin, newEpoch, reason)`, fired from `onPeerActivated` (`utils/federationPeerActivation.ts`) **after an admin-authenticated re-peer**, keyed to the confirmed epoch change (design §6). It runs **before** the mutation-log re-sync in `onPeerActivated` so re-sync repopulates onto a clean slate, and it runs **outside any transaction** (`tombstoneUser` opens its own; better-sqlite3 throws on a nested `BEGIN`). diff --git a/docs/systems/social.md b/docs/systems/social.md index c934869b..d2ca8ad0 100644 --- a/docs/systems/social.md +++ b/docs/systems/social.md @@ -293,6 +293,7 @@ As of 2026-04-25, the sender's home server owns the entire federated friend-add 1. **Parse target.** If `body.username` contains no `@`, or the domain after `@` normalizes to this server's own host, fall through to the local-only path (unchanged). 2. **resolveOriginFromHostname(targetDomain)** — resolves the target peer's full origin URL. Prefers a stored `federation_peers` row matching the typed host; falls back to mirroring `getOurOrigin()`'s scheme. Returns null → 400 `invalid_target_domain`. +2a. **Limbo-window guard → 409 `peer_reset_pending`.** O(1) point lookup on the `federation_reset_events` origin PRIMARY KEY: if an **unresolved** row exists for `peerOrigin` (`origin = peerOrigin AND resolved_at IS NULL`), the peer was reset-detected (wipe-and-reinstall) but the admin has not yet re-peered — the local friendship/stub graph is still bound to the dead incarnation. Return 409 `peer_reset_pending` instead of the confusing `already_friends` (stale friendship) or `peer_rejected` (the `needs_attention` peer would otherwise trip `ensurePeered`). `peerOrigin` is the exact string `markPeerReset` journals (the peer's `federation_peers.origin`), so the match is a single indexed lookup; no reset in progress → one indexed miss → the normal path proceeds unchanged. See `docs/systems/federation.md` (instance-epoch self-healing) and the design spec §5.3. The equivalent guard runs on federated DM-create (`POST /api/dm`, `dm.ts`). 3. **Authority defense.** If the calling user's `homeInstance` is set and does not normalize to this server's own host (checked via `normalizeOriginForCompare`), return 403 `not_authoritative_for_sender`. Prevents replicated/federated users from queueing relay events the home server isn't authoritative for. Runs before peering to fail fast. 4. **ensurePeered(peerOrigin)** — blocks on the result. Status → HTTP mapping: - `'active'` → continue diff --git a/packages/server/src/routes/dm.peerResetPending.test.ts b/packages/server/src/routes/dm.peerResetPending.test.ts new file mode 100644 index 00000000..40563c65 --- /dev/null +++ b/packages/server/src/routes/dm.peerResetPending.test.ts @@ -0,0 +1,165 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import Fastify, { type FastifyInstance } from 'fastify'; +import Database from 'better-sqlite3'; +import { drizzle } from 'drizzle-orm/better-sqlite3'; +import { eq } from 'drizzle-orm'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import * as schema from '../db/schema.js'; +import { setWorkerId } from '../utils/snowflake.js'; + +setWorkerId(1); + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +type TestDb = ReturnType>; +let sqlite: Database.Database; +let testDb: TestDb; +const currentUserId = 'user-A'; + +vi.mock('../db/index.js', () => ({ + getDb: () => testDb, + schema, +})); + +vi.mock('../utils/auth.js', () => ({ + authenticate: async (req: { userId?: string }) => { + req.userId = currentUserId; + }, +})); + +vi.mock('../ws/handler.js', () => ({ + connectionManager: { + sendToUser: vi.fn(), + sendToDmMembers: vi.fn(), + sendToAdmins: vi.fn(), + getAllOnlineUserIds: () => [], + }, +})); + +vi.mock('../utils/federationOutbox.js', async () => { + const actual = await vi.importActual('../utils/federationOutbox.js'); + return { + ...actual, + isFederationRelayEnabled: () => true, + queueDmCloseRelay: vi.fn(), + sendTypingRelay: vi.fn(), + queueDmRelay: vi.fn(), + queueOutboxEvent: vi.fn(), + appendMutationLog: vi.fn(), + }; +}); + +function applyMigrations(db: Database.Database): void { + const migrationsDir = path.resolve(__dirname, '../../drizzle'); + const files = fs.readdirSync(migrationsDir).filter(f => f.endsWith('.sql')).sort(); + for (const f of files) { + const sqlText = fs.readFileSync(path.join(migrationsDir, f), 'utf8'); + const statements = sqlText.split(/-->\s*statement-breakpoint/); + for (const stmt of statements) { + const clean = stmt.trim(); + if (clean) db.exec(clean); + } + } +} + +function seedCaller(): void { + testDb.insert(schema.users).values({ + id: 'user-A', + username: 'alice', + displayName: 'Alice', + passwordHash: 'x', + homeUserId: 'user-A', + homeInstance: null, + createdAt: Date.now(), + }).run(); +} + +/** The reset peer's persistent row (still present during the limbo window — only + * deleted on admin Re-peer). Its `origin` is the exact string markPeerReset journals. */ +function seedPeer(): void { + testDb.insert(schema.federationPeers).values({ + id: 'peer-remote', + origin: 'https://remote.example', + hmacSecret: 'secret', + status: 'needs_attention', + needsAttentionReason: 'peer_reset_detected', + peerInstanceId: 'dead-epoch', + observedPeerInstanceId: 'new-epoch', + createdAt: Date.now(), + }).run(); +} + +function seedResetEvent(resolvedAt: number | null): void { + testDb.insert(schema.federationResetEvents).values({ + origin: 'https://remote.example', + deadEpoch: 'dead-epoch', + newEpoch: resolvedAt === null ? null : 'new-epoch', + detectedAt: Date.now(), + resolvedAt, + stubCount: 1, + orphanedAccountCount: 0, + }).run(); +} + +async function buildApp(): Promise { + const app = Fastify({ logger: false }); + const { dmRoutes } = await import('./dm.js'); + await app.register(dmRoutes); + await app.ready(); + return app; +} + +describe('POST /api/dm — limbo-window peer_reset_pending guard', () => { + beforeEach(() => { + sqlite = new Database(':memory:'); + testDb = drizzle(sqlite, { schema }); + applyMigrations(sqlite); + seedCaller(); + seedPeer(); + }); + + it('returns 409 peer_reset_pending when creating a federated DM to a reset-pending origin', async () => { + seedResetEvent(null); + + const app = await buildApp(); + const res = await app.inject({ + method: 'POST', + url: '/api/dm', + payload: { homeUserId: 'remote-bob', homeInstance: 'https://remote.example' }, + }); + + expect(res.statusCode).toBe(409); + expect(res.json().error).toBe('peer_reset_pending'); + // No stub created and no DM channel created for the reset-pending peer. + expect(testDb.select().from(schema.dmChannels).all()).toHaveLength(0); + expect(testDb.select().from(schema.users).where(eq(schema.users.homeUserId, 'remote-bob')).all()).toHaveLength(0); + }); + + it('proceeds normally when the reset event is RESOLVED', async () => { + seedResetEvent(Date.now()); + + const app = await buildApp(); + const res = await app.inject({ + method: 'POST', + url: '/api/dm', + payload: { homeUserId: 'remote-bob', homeInstance: 'https://remote.example' }, + }); + + expect(res.statusCode).toBe(201); + expect(res.json().federatedId).toMatch(/^[a-f0-9]{32}$/); + }); + + it('proceeds normally when NO reset event exists for the origin', async () => { + const app = await buildApp(); + const res = await app.inject({ + method: 'POST', + url: '/api/dm', + payload: { homeUserId: 'remote-bob', homeInstance: 'https://remote.example' }, + }); + + expect(res.statusCode).toBe(201); + expect(res.json().federatedId).toMatch(/^[a-f0-9]{32}$/); + }); +}); diff --git a/packages/server/src/routes/dm.ts b/packages/server/src/routes/dm.ts index c800f8bf..5d925999 100644 --- a/packages/server/src/routes/dm.ts +++ b/packages/server/src/routes/dm.ts @@ -47,6 +47,7 @@ import { normalizeIconForWire, } from '../utils/federationOutbox.js'; import { getOurOrigin, canonicalizeHomeInstance } from '../utils/federationAuth.js'; +import { resolveOriginFromHostname } from '../utils/federationOriginResolve.js'; import type { FederationRelayEvent } from '@backspace/shared'; import { resolveLocalUser, resolveOrCreateReplicatedUser } from './federation.js'; @@ -928,6 +929,40 @@ export async function dmRoutes(app: FastifyInstance): Promise { let targetUser: typeof schema.users.$inferSelect | undefined; if (homeUserId && homeInstance) { + // Limbo-window guard (federation instance-epoch self-healing §5.3). + // If the target's home instance was reset-detected but the admin has not yet + // re-peered, an UNRESOLVED `federation_reset_events` row exists for its origin. + // Creating a DM now would bind to stale, dead-incarnation identity state, so + // surface a clear `peer_reset_pending` instead of silently forming a doomed + // channel. Checked BEFORE stub creation so no un-flagged stub is left behind. + // + // The journal is keyed by the peer's `federation_peers.origin` (the exact string + // `markPeerReset` stores). `resolveOriginFromHostname` returns that stored origin + // verbatim, giving an O(1) point lookup on the origin PRIMARY KEY; the common case + // (no reset) is a single indexed miss and the normal path proceeds unchanged. + const canon = canonicalizeHomeInstance(homeInstance); + let peerOrigin: string | null = null; + if (canon) { + try { + peerOrigin = resolveOriginFromHostname(new URL(canon).host); + } catch { + peerOrigin = null; + } + } + if (peerOrigin) { + const pendingReset = db + .select({ origin: schema.federationResetEvents.origin }) + .from(schema.federationResetEvents) + .where(and( + eq(schema.federationResetEvents.origin, peerOrigin), + isNull(schema.federationResetEvents.resolvedAt), + )) + .get(); + if (pendingReset) { + return reply.code(409).send({ error: 'peer_reset_pending', statusCode: 409 }); + } + } + // Federated identity: resolve or create a replicated user stub targetUser = resolveOrCreateReplicatedUser(homeUserId, homeInstance, db) ?? undefined; } else if (userId && typeof userId === 'string') { diff --git a/packages/server/src/routes/social.federated.test.ts b/packages/server/src/routes/social.federated.test.ts index 2318fa0f..f26dccea 100644 --- a/packages/server/src/routes/social.federated.test.ts +++ b/packages/server/src/routes/social.federated.test.ts @@ -457,3 +457,100 @@ describe('POST /api/social/requests — federated branch (authority + self-frien expect(body.requestId).toBe('incoming-req'); }); }); + +describe('POST /api/social/requests — federated branch (limbo-window peer_reset_pending)', () => { + beforeEach(() => { + seedSelf(); + resolveOriginFromHostnameMock.mockReturnValue('https://orbit.test'); + }); + + function seedResetEvent(resolvedAt: number | null): void { + testDb.insert(schema.federationResetEvents).values({ + origin: 'https://orbit.test', + deadEpoch: 'dead-epoch', + newEpoch: resolvedAt === null ? null : 'new-epoch', + detectedAt: Date.now(), + resolvedAt, + stubCount: 1, + orphanedAccountCount: 0, + }).run(); + } + + it('returns 409 peer_reset_pending when an UNRESOLVED reset event exists for the target origin', async () => { + seedResetEvent(null); + // Even a stale friendship must NOT surface as `already_friends` during the limbo window. + testDb.insert(schema.users).values({ + id: 'stub-alice', + username: 'remote-alice@orbit.test', + displayName: 'Alice', + passwordHash: '!federation-replicated', + status: 'offline', + isAdmin: 0, + homeInstance: 'orbit.test', + homeUserId: 'remote-alice-old', + createdAt: Date.now(), + }).run(); + testDb.insert(schema.friends).values({ + userId: CALLER_ID, + friendId: 'stub-alice', + createdAt: Date.now(), + }).run(); + + const app = await buildApp(); + const res = await app.inject({ + method: 'POST', + url: '/api/social/requests', + payload: { username: 'alice@orbit.test' }, + }); + + expect(res.statusCode).toBe(409); + expect(JSON.parse(res.body).error).toBe('peer_reset_pending'); + // Short-circuits before peering/lookup — neither is consulted. + expect(ensurePeeredMock).not.toHaveBeenCalled(); + expect(lookupRemoteUserMock).not.toHaveBeenCalled(); + // No new request row created. + expect(testDb.select().from(schema.friendRequests).all()).toHaveLength(0); + }); + + it('proceeds normally when the reset event is RESOLVED (resolved_at set)', async () => { + seedResetEvent(Date.now()); + ensurePeeredMock.mockResolvedValue({ status: 'active', peerId: 'peer-1' }); + lookupRemoteUserMock.mockResolvedValue({ + ok: true, + homeUserId: 'remote-alice', + username: 'alice', + profile: { displayName: 'Alice', avatar: null, avatarColor: 'mint', banner: null, bio: null }, + }); + + const app = await buildApp(); + const res = await app.inject({ + method: 'POST', + url: '/api/social/requests', + payload: { username: 'alice@orbit.test' }, + }); + + expect(res.statusCode).toBe(201); + expect(ensurePeeredMock).toHaveBeenCalled(); + expect(JSON.parse(res.body).success).toBe(true); + }); + + it('proceeds normally when NO reset event exists for the origin', async () => { + ensurePeeredMock.mockResolvedValue({ status: 'active', peerId: 'peer-1' }); + lookupRemoteUserMock.mockResolvedValue({ + ok: true, + homeUserId: 'remote-alice', + username: 'alice', + profile: { displayName: 'Alice', avatar: null, avatarColor: 'mint', banner: null, bio: null }, + }); + + const app = await buildApp(); + const res = await app.inject({ + method: 'POST', + url: '/api/social/requests', + payload: { username: 'alice@orbit.test' }, + }); + + expect(res.statusCode).toBe(201); + expect(ensurePeeredMock).toHaveBeenCalled(); + }); +}); diff --git a/packages/server/src/routes/social.ts b/packages/server/src/routes/social.ts index 39b07331..c89b0ccd 100644 --- a/packages/server/src/routes/social.ts +++ b/packages/server/src/routes/social.ts @@ -1,5 +1,5 @@ import type { FastifyInstance, FastifyRequest, FastifyReply } from 'fastify'; -import { eq, and, or, ne, like, sql, inArray } from 'drizzle-orm'; +import { eq, and, or, ne, like, sql, inArray, isNull } from 'drizzle-orm'; import { getDb, getRawDb, schema } from '../db/index.js'; import { authenticate } from '../utils/auth.js'; import { generateSnowflake } from '../utils/snowflake.js'; @@ -183,6 +183,32 @@ async function handleFederatedFriendRequest( return reply.code(400).send({ error: 'invalid_target_domain', statusCode: 400, domain: targetDomain }); } + // 1a. Limbo-window guard (federation instance-epoch self-healing §5.3). + // If this peer's home instance was reset-detected but the admin has not yet + // re-peered, an UNRESOLVED `federation_reset_events` row exists for its origin + // (the peer sits in `needs_attention`, its local friendship/stub graph still + // bound to the dead incarnation). Without this guard the re-add would surface + // a confusing `already_friends` (stale friendship) or `peer_rejected` (the + // needs_attention peer) — neither of which tells the user what to do. Return a + // clear `peer_reset_pending` instead. + // + // `resolveOriginFromHostname` returns the peer's stored `federation_peers.origin` + // verbatim, which is exactly the string `markPeerReset` journals as + // `federation_reset_events.origin` (its PRIMARY KEY), so this is an O(1) indexed + // point lookup. The common case — no reset in progress — is a single indexed miss + // and the normal path proceeds unchanged. + const pendingReset = db + .select({ origin: schema.federationResetEvents.origin }) + .from(schema.federationResetEvents) + .where(and( + eq(schema.federationResetEvents.origin, peerOrigin), + isNull(schema.federationResetEvents.resolvedAt), + )) + .get(); + if (pendingReset) { + return reply.code(409).send({ error: 'peer_reset_pending', statusCode: 409 }); + } + // 2. ensurePeered — block until 'active', or surface peer status as error const peering = await ensurePeered(peerOrigin, { kind: 'user_action', From 769ed6431450f3df63dd8463ae82cbb4853b5fc8 Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Thu, 2 Jul 2026 00:20:43 +0200 Subject: [PATCH 11/12] docs(federation): document instance-epoch self-healing (Phase 1) --- docs/systems/api.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/systems/api.md b/docs/systems/api.md index 2485a2c6..199d86e4 100644 --- a/docs/systems/api.md +++ b/docs/systems/api.md @@ -227,10 +227,12 @@ Permissions checked: CONNECT, SPEAK, STREAM (space channels). DM calls: always f ## Instance (`routes/instance.ts`) — public ``` -GET /instance/info → { name, version, registrationOpen, federatedRegistrationOpen, sourceCodeUrl, commit } +GET /instance/info → { name, version, registrationOpen, federatedRegistrationOpen, instanceId, sourceCodeUrl, commit } ``` `federatedRegistrationOpen` is a UX hint consumed by the Connections add-instance pre-flight (see `client-federation.md`). The 403 from `POST /auth/register` remains the security boundary. +`instanceId` (`InstanceInfoResponse.instanceId`, `string`) is this instance's persistent **epoch** — the incarnation UUID minted once by `ensureDefaults` and stable across restarts (see `database.md → Instance Settings`). It is served here (unauthenticated, credential-free) purely as a **detection** signal: `probePeerReachable` reads it to observe that a peer behind a known origin has been factory-reset (a changed epoch). It is **never** written to a peer's trusted baseline from this channel — only the authenticated `/federation/epoch`, relay envelope, and handshake do that. See `federation.md` "Instance Epoch". + `sourceCodeUrl` (`string`) and `commit` (`string | null`) implement the **AGPL-3.0 § 13 network-use source offer**: every network user (and federated peer) can obtain the Corresponding Source of the exact version this instance is running. `sourceCodeUrl` comes from `config.sourceCodeUrl` (env `BACKSPACE_SOURCE_URL`, default `https://github.com/TheZwiss/backspace`) — operators who modify Backspace MUST set it to their fork's source. `commit` comes from `config.commit` (env `BACKSPACE_COMMIT`, injected at Docker build via `deploy.sh --build-arg`; `null` in local dev). The web client surfaces `sourceCodeUrl`/`version` via the `SourceCodeLink` component on settings sidebars and the pre-auth login/register pages; the desktop app exposes it via the tray + app menus ("Source code (AGPL)") and the native About panel. ## Settings (`routes/settings.ts`) @@ -318,10 +320,10 @@ type InviteRedemption = { ## Federation (`routes/federation.ts`) ``` POST /federation/peer/initiate (admin) { remoteOrigin } → peer created -POST /federation/peer/accept (public, IP rate-limited 10/min) { sourceOrigin, challenge, hmacSecret, instanceName?, approvalToken? } → accepted (200) | queued (202 + { approvalToken }) +POST /federation/peer/accept (public, IP rate-limited 10/min) { sourceOrigin, challenge, hmacSecret, instanceName?, instanceId?, approvalToken? } → { accepted, instanceName, instanceId } (200) | queued (202 + { approvalToken }) GET /federation/peers (admin) → { peers[] } (no secrets) DELETE /federation/peers/:id (admin) → { success } + outbox cleanup -POST /federation/relay (HMAC-signed S2S) FederationRelayRequest → { accepted[], rejected[] } +POST /federation/relay (HMAC-signed S2S) FederationRelayRequest (+ sourceInstanceId?) → { accepted[], rejected[] } POST /federation/sync (HMAC-signed S2S) { sinceTimestamp, limit?, dmChannelId?, federatedId?, contextType? } → { events[], hasMore, checkpoint } POST /federation/users/lookup (HMAC-signed S2S, rate-limited 60/min/peer) { username } → { found, user? } POST /federation/epoch (HMAC-signed S2S, HMAC-signed response) {} → { instanceId } @@ -329,6 +331,8 @@ POST /federation/epoch (HMAC-signed S2S, HMAC-signed response) {} **`POST /api/federation/peer/accept`** — public, IP-rate-limited. Optional `approvalToken` (64-hex) on the request body proves mutual admin approval; required to promote an `awaiting_approval` row to `active` when the receiver has `autoAcceptPeering=0`. The receiver returns it in the 202 body when queueing the request for admin review (`{ queued: true, message, approvalToken }`); the initiator stores it and the receiver's `/approve` later forwards it back. See `federation.md` §1 "Approval Token Verification" for the full lifecycle and threat model. +**Handshake epoch exchange.** The handshake carries the **instance epoch** bidirectionally, mirroring `instanceName`: the request body's `instanceId` is the initiator's epoch (written to `federation_peers.peer_instance_id` on every authenticated activation path), and the 200 response body's `instanceId` is the responder's epoch (persisted by the initiator alongside `status='active'`). Older peers omit the field; the column stays `null` until the epoch-refresh/relay backstop fills it. Both are authenticated baselines — never overwritten by the unauthenticated `/instance/info` probe. **`FederationRelayRequest.sourceInstanceId`** stamps the sender's current epoch on every relay; because the whole body is HMAC-verified, a valid relay authentically carries the sender's incarnation id and populates `peer_instance_id` when null (fast-path baseline). See `federation.md` "Instance Epoch". + **`POST /api/federation/users/lookup`** — HMAC-authenticated S2S endpoint. Resolves a username on this instance to its canonical `(homeUserId, profile snapshot)`. Used by the cross-instance friend-add flow on the sender's home server before queuing a `friend_request_create` event. Responds to native, non-deleted users only; ignores `discoverable`. Returns `{ found: false, code: 'user_not_found' }` for stubs, tombstoned users, or unknown handles. See `federation.md` §1 "S2S User Lookup" for the full contract. **`POST /api/federation/epoch`** — HMAC-authenticated S2S endpoint returning this instance's persistent epoch (`{ instanceId }`). The **request** is HMAC-signed (only a peer holding the shared secret may call it; unknown/revoked peers → 403, bad signature → 401, missing headers → 400) **and the response body is HMAC-signed** with the same secret (`X-Federation-Signature/Timestamp/Nonce` response headers), so the caller can verify the epoch before writing it as the peer's trusted baseline (`federation_peers.peer_instance_id`). The value is already public via `/instance/info`; signing is for baseline-integrity, not confidentiality. Caller: `fetchPeerEpoch(peer)` (`utils/federationEpoch.ts`), which fails safe — 404 (not-yet-upgraded peer), bad/absent response signature, or network/timeout all return `null` (retry next tick). Populates the epoch baseline deterministically via the bounded periodic epoch-refresh. See `federation.md` "Instance Epoch" §3.2. From d8fec009055b66cec58bc53dbaf22f8196bced41 Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Thu, 2 Jul 2026 00:34:33 +0200 Subject: [PATCH 12/12] =?UTF-8?q?feat(federation):=20detect=20peer=20reset?= =?UTF-8?q?=20on=20needs=5Fattention=20peers=20(=C2=A74.1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A reset peer can reach needs_attention via the auth-failure path (HTTP up, 401/403 from a new incarnation crossing AUTH_FAILURE_THRESHOLD) without ever passing through unreachable, so the unreachable-only recovery probe never observes its epoch change and no reset journal is created — leaving a later manual Re-peer with nothing to heal. Add detectResetOnNeedsAttentionPeers() to the 15-minute health-check tick: probe needs_attention peers with a non-null baseline (excluding those already peer_reset_detected) and call markPeerReset on an observed epoch mismatch. Detection only — never recovers a needs_attention peer to active; baseline (peer_instance_id) and hmac_secret untouched. --- docs/systems/federation.md | 5 +- .../src/utils/federationRecovery.test.ts | 84 +++++++++++++++++++ .../server/src/utils/federationRecovery.ts | 61 +++++++++++++- packages/server/src/utils/federationWorker.ts | 13 ++- 4 files changed, 159 insertions(+), 4 deletions(-) diff --git a/docs/systems/federation.md b/docs/systems/federation.md index eb87d518..836b884e 100644 --- a/docs/systems/federation.md +++ b/docs/systems/federation.md @@ -327,9 +327,10 @@ In a single transaction, `markPeerReset`: **Idempotent / double-reset:** if an *unresolved* `federation_reset_events` row already exists for the origin (the peer reset again before an admin resolved the first), the original `dead_epoch` and `detected_at` are **preserved** (that is the incarnation whose users are already snapshotted) — only the summary counts are refreshed. `dead_epoch` is never overwritten on an unresolved row. A prior *resolved* reset starts a fresh journal entry. -**Detection sources (both wired in this feature):** +**Detection sources (all three wired in this feature):** - **Inbound handshake** — `/peer/accept` landing on an `active`/`needs_attention` row (`routes/federation.ts`): before the idempotent-200 return, `if (reqInstanceId && existing.peerInstanceId && reqInstanceId !== existing.peerInstanceId) markPeerReset(...)`. The guard still returns 200 and **still does not rekey** — the anti-hijack property is preserved verbatim; detection is layered on top. -- **Reachability probe** — `probePeerReachable()` (`utils/federationRecovery.ts`) now parses `instanceId` from the `/api/instance/info` response (returning `{ reachable, instanceId }`; a missing/unparseable epoch is `null`, never an error). The shared decision helper `recoverOrDetectReset(peer, result)` — used by both the background recovery tick (`processRecoveryTick`) and the manual recheck endpoint — routes to `markPeerReset` (returning `'reset_detected'`) when the peer has a non-null `peer_instance_id` and the probed epoch differs, and **does NOT call `markPeerRecovered`**. Rationale: a genuinely reset peer's HMAC secret is desynced, so flipping it back to `active` via a reachability probe would resume relay against a dead secret. Only when the probed epoch matches the baseline (or the baseline is null / epoch unknown) does the normal recovery-to-active path run. The manual recheck endpoint returns `{ recovered: false, status: 'needs_attention' }` on `'reset_detected'`. +- **Reachability probe (`unreachable` peers)** — `probePeerReachable()` (`utils/federationRecovery.ts`) now parses `instanceId` from the `/api/instance/info` response (returning `{ reachable, instanceId }`; a missing/unparseable epoch is `null`, never an error). The shared decision helper `recoverOrDetectReset(peer, result)` — used by both the background recovery tick (`processRecoveryTick`) and the manual recheck endpoint — routes to `markPeerReset` (returning `'reset_detected'`) when the peer has a non-null `peer_instance_id` and the probed epoch differs, and **does NOT call `markPeerRecovered`**. Rationale: a genuinely reset peer's HMAC secret is desynced, so flipping it back to `active` via a reachability probe would resume relay against a dead secret. Only when the probed epoch matches the baseline (or the baseline is null / epoch unknown) does the normal recovery-to-active path run. The manual recheck endpoint returns `{ recovered: false, status: 'needs_attention' }` on `'reset_detected'`. +- **Health-tick probe (`needs_attention` peers)** — `detectResetOnNeedsAttentionPeers()` (`utils/federationRecovery.ts`), called from `processHealthCheckTick` on the 15-minute tick, closes design §4.1's remaining sub-case. A reset peer can reach `needs_attention` via the **auth-failure path** — its HTTP is up but returns 401/403 because the new incarnation has no peer row for us, so `consecutive_auth_failures` crosses `AUTH_FAILURE_THRESHOLD` — **without ever transitioning through `unreachable`**. The `unreachable`-only recovery probe therefore never observes its epoch change, so no journal is ever created; a later manual Re-peer would then run `healResetIncarnation` with no journal row → no heal → the split-brain persists. This pass selects peers with `status='needs_attention'` AND `peer_instance_id IS NOT NULL` AND `needs_attention_reason` not already `peer_reset_detected` (those already carry a journal), probes each (`probePeerReachable`, one `/instance/info` GET per qualifying peer per tick), and calls `markPeerReset` on an observed epoch mismatch. **Detection only:** unlike `recoverOrDetectReset`, it NEVER flips a `needs_attention` peer to `active` (a match / unknown / unreachable result is a pure no-op) — that peer's secret is desynced and only an admin-authenticated re-peer restores trust. It touches neither `peer_instance_id` nor `hmac_secret`. Legacy peers advertise no epoch (`instanceId` null), so detection requires a non-null observed epoch differing from a non-null stored baseline — legacy peers never trigger it, and the existing `auth_failures → needs_attention → manual Reset` path continues unchanged for them. diff --git a/packages/server/src/utils/federationRecovery.test.ts b/packages/server/src/utils/federationRecovery.test.ts index db91d1e1..1cea5e18 100644 --- a/packages/server/src/utils/federationRecovery.test.ts +++ b/packages/server/src/utils/federationRecovery.test.ts @@ -150,4 +150,88 @@ describe('federationRecovery primitives', () => { // A reset peer must NOT be recovered to active. expect(onPeerActivated).not.toHaveBeenCalled(); }); + + // ── detectResetOnNeedsAttentionPeers (design §4.1) ───────────────────────── + // Closes the auth-failure sub-case: a reset peer whose HTTP is up (returning + // 401/403 because the new incarnation has no peer row for us) crosses + // AUTH_FAILURE_THRESHOLD and lands in `needs_attention` WITHOUT ever passing + // through `unreachable`, so the unreachable-only recovery probe never observes + // its epoch change. This pass probes those peers too — detection ONLY, never a + // recover-to-active. + function seedNeedsAttention(id: string, reason: string | null, peerInstanceId: string | null): void { + testDb.insert(schema.federationPeers).values({ + id, origin: 'https://peer.example', hmacSecret: 'secret', + status: 'needs_attention', needsAttentionReason: reason, + peerInstanceId, consecutiveFailures: 0, + lastSyncedAt: Date.now(), createdAt: Date.now(), + }).run(); + } + + it('detectResetOnNeedsAttentionPeers flags an auth-failure peer whose epoch changed (detection only)', async () => { + seedNeedsAttention('peer-na', 'auth_failures', 'E0'); + // A pure replicated stub belonging to the dead incarnation (bare-domain home). + testDb.insert(schema.users).values({ + id: 'stub-1', username: 'carol', displayName: 'carol', + passwordHash: '!federation-replicated', homeInstance: 'peer.example', + createdAt: Date.now(), + }).run(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('{"instanceId":"E1"}', { status: 200 })); + + const { detectResetOnNeedsAttentionPeers } = await import('./federationRecovery.js'); + await detectResetOnNeedsAttentionPeers(); + + const row = testDb.select().from(schema.federationPeers) + .where(eq(schema.federationPeers.id, 'peer-na')).get()!; + expect(row.status).toBe('needs_attention'); // NOT flipped to active + expect(row.needsAttentionReason).toBe('peer_reset_detected'); + expect(row.observedPeerInstanceId).toBe('E1'); // observed epoch recorded + expect(row.peerInstanceId).toBe('E0'); // trusted baseline untouched + expect(row.hmacSecret).toBe('secret'); // secret untouched + + const journal = testDb.select().from(schema.federationResetEvents) + .where(eq(schema.federationResetEvents.origin, 'https://peer.example')).get()!; + expect(journal.deadEpoch).toBe('E0'); + expect(journal.resolvedAt).toBeNull(); + + const stub = testDb.select().from(schema.users) + .where(eq(schema.users.id, 'stub-1')).get()!; + expect(stub.federationHealPending).toBe(1); // dead incarnation snapshotted + + expect(onPeerActivated).not.toHaveBeenCalled(); // detection only + }); + + it('detectResetOnNeedsAttentionPeers is a no-op when the probed epoch matches the baseline', async () => { + seedNeedsAttention('peer-same', 'auth_failures', 'E0'); + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('{"instanceId":"E0"}', { status: 200 })); + + const { detectResetOnNeedsAttentionPeers } = await import('./federationRecovery.js'); + await detectResetOnNeedsAttentionPeers(); + + const row = testDb.select().from(schema.federationPeers) + .where(eq(schema.federationPeers.id, 'peer-same')).get()!; + expect(row.status).toBe('needs_attention'); + expect(row.needsAttentionReason).toBe('auth_failures'); // unchanged + expect(testDb.select().from(schema.federationResetEvents).all()).toHaveLength(0); + expect(onPeerActivated).not.toHaveBeenCalled(); + }); + + it('detectResetOnNeedsAttentionPeers skips peers already flagged peer_reset_detected (no probe)', async () => { + seedNeedsAttention('peer-done', 'peer_reset_detected', 'E0'); + const spy = vi.spyOn(globalThis, 'fetch'); + + const { detectResetOnNeedsAttentionPeers } = await import('./federationRecovery.js'); + await detectResetOnNeedsAttentionPeers(); + + expect(spy).not.toHaveBeenCalled(); // already journaled — not re-probed + }); + + it('detectResetOnNeedsAttentionPeers skips peers with a null baseline (nothing to compare)', async () => { + seedNeedsAttention('peer-nobase', 'auth_failures', null); + const spy = vi.spyOn(globalThis, 'fetch'); + + const { detectResetOnNeedsAttentionPeers } = await import('./federationRecovery.js'); + await detectResetOnNeedsAttentionPeers(); + + expect(spy).not.toHaveBeenCalled(); // no trusted baseline → cannot detect a change + }); }); diff --git a/packages/server/src/utils/federationRecovery.ts b/packages/server/src/utils/federationRecovery.ts index c9734173..aec3eace 100644 --- a/packages/server/src/utils/federationRecovery.ts +++ b/packages/server/src/utils/federationRecovery.ts @@ -1,6 +1,6 @@ import { getDb } from '../db/index.js'; import * as schema from '../db/schema.js'; -import { eq } from 'drizzle-orm'; +import { and, eq, isNotNull, isNull, ne, or } from 'drizzle-orm'; import { onPeerActivated } from './federationPeerActivation.js'; import { markPeerReset } from './federationReset.js'; @@ -103,3 +103,62 @@ export async function recoverOrDetectReset( await markPeerRecovered(peer.id); return 'recovered'; } + +/** + * Detection-only epoch probe for peers already parked in `needs_attention` + * (design §4.1). Runs on the 15-minute health-check tick. + * + * The gap this closes: a reset peer can reach `needs_attention` via the + * AUTH-FAILURE path — its HTTP is up and returning 401/403 because the new + * incarnation has no peer row for us, so `consecutive_auth_failures` crosses + * `AUTH_FAILURE_THRESHOLD` — WITHOUT ever transitioning through `unreachable`. + * The `unreachable`-only recovery probe (`processRecoveryTick`) therefore never + * sees such a peer, so its epoch change is never observed and no + * `federation_reset_events` journal is ever created. A later manual admin + * Re-peer would then run `healResetIncarnation` with no journal row → no heal → + * the stale-friendship / split-DM split-brain persists for this sub-case. This + * pass probes those peers so the journal is created at detection time. + * + * **Detection only — never a recover-to-active.** Unlike `recoverOrDetectReset`, + * this NEVER flips a peer to `active`: a `needs_attention` peer's HMAC secret is + * desynced, so a matching or unknown epoch means "still broken, still needs an + * admin," not "recovered." On an observed epoch mismatch it calls + * `markPeerReset` (snapshot + journal + admin notify) and nothing else; the + * trusted baseline (`peer_instance_id`) and `hmac_secret` are left untouched. + * On a match / unknown / unreachable result it does nothing at all. + * + * Candidate set (deliberately small — one `/instance/info` GET per peer per + * tick): `status='needs_attention'` AND `peer_instance_id IS NOT NULL` (a null + * baseline has nothing to compare against) AND the reason is not already + * `peer_reset_detected` (those peers already carry a journal — re-probing would + * be wasted work). Peers whose reason is `auth_failures` or NULL qualify. + */ +export async function detectResetOnNeedsAttentionPeers(signal?: AbortSignal): Promise { + const db = getDb(); + const peers = db + .select({ + id: schema.federationPeers.id, + origin: schema.federationPeers.origin, + peerInstanceId: schema.federationPeers.peerInstanceId, + }) + .from(schema.federationPeers) + .where(and( + eq(schema.federationPeers.status, 'needs_attention'), + isNotNull(schema.federationPeers.peerInstanceId), + or( + isNull(schema.federationPeers.needsAttentionReason), + ne(schema.federationPeers.needsAttentionReason, 'peer_reset_detected'), + ), + )) + .all(); + + for (const peer of peers) { + const result = await probePeerReachable(peer.origin, signal); + // Detection fires ONLY on a reachable peer advertising a non-null epoch that + // differs from the trusted baseline. Everything else (unreachable, unknown + // epoch, or a matching epoch) is a no-op — no recover-to-active from here. + if (result.reachable && result.instanceId && peer.peerInstanceId && result.instanceId !== peer.peerInstanceId) { + markPeerReset(peer.id, peer.origin, peer.peerInstanceId, result.instanceId); + } + } +} diff --git a/packages/server/src/utils/federationWorker.ts b/packages/server/src/utils/federationWorker.ts index ecfbd02b..90164a26 100644 --- a/packages/server/src/utils/federationWorker.ts +++ b/packages/server/src/utils/federationWorker.ts @@ -12,7 +12,7 @@ import { connectionManager } from '../ws/handler.js'; import { generateThumbnail } from './thumbnail.js'; import type { FederationRelayRequest, FederationRelayResponse, FederationRelayEvent } from '@backspace/shared'; import { startupBootstrapSync, onPeerDeactivated } from './federationPeerActivation.js'; -import { probePeerReachable, recoverOrDetectReset } from './federationRecovery.js'; +import { probePeerReachable, recoverOrDetectReset, detectResetOnNeedsAttentionPeers } from './federationRecovery.js'; import { backfillReplicatedProfileAssets } from '../routes/federation.js'; import { invokePermanentFailureCallback } from './federationRollback.js'; import { refreshPeerEpochs, getInstanceId } from './federationEpoch.js'; @@ -1180,6 +1180,17 @@ async function processHealthCheckTick(): Promise { // relay/user activity. Best-effort — a failed fetch is a benign no-op retried // next tick, so it never disturbs the rest of the health-check work. await refreshPeerEpochs().catch(() => {}); + + // ── Reset detection for needs_attention peers (design §4.1) ───────────────── + // A reset peer can land in `needs_attention` via the auth-failure path (HTTP + // up, 401/403 from a new incarnation) WITHOUT ever passing through + // `unreachable`, so the unreachable-only recovery probe never observes its + // epoch change. Probe those peers here so a reset journal is created at + // detection time (otherwise a later manual Re-peer heals nothing). Detection + // ONLY — never flips a needs_attention peer to active. Best-effort: a failure + // is a benign no-op retried next tick and must not disturb the rest of the tick. + // No shared abort signal — probePeerReachable carries its own 10s timeout. + await detectResetOnNeedsAttentionPeers().catch(() => {}); } // ─── Federated Call Health Sweep ────────────────────────────────────────────