diff --git a/docs/systems/federation.md b/docs/systems/federation.md index d44b5ab9..69f8f1c2 100644 --- a/docs/systems/federation.md +++ b/docs/systems/federation.md @@ -333,6 +333,22 @@ In a single transaction, `markPeerReset`: Legacy peers advertise no epoch (`instanceId` null), so detection requires a non-null observed epoch differing from a non-null stored baseline — legacy peers never trigger it, and the existing `auth_failures → needs_attention → manual Reset` path continues unchanged for them. +### Data Self-Heal (`healResetIncarnation` — `utils/federationReset.ts`) + +Detection (`markPeerReset`) only snapshots + journals + notifies; it destroys nothing. The actual heal is `healResetIncarnation(origin, newEpoch, reason)`, fired from `onPeerActivated` (`utils/federationPeerActivation.ts`) **after an admin-authenticated re-peer**, keyed to the confirmed epoch change (design §6). It runs **before** the mutation-log re-sync in `onPeerActivated` so re-sync repopulates onto a clean slate, and it runs **outside any transaction** (`tombstoneUser` opens its own; better-sqlite3 throws on a nested `BEGIN`). + +**Two mandatory guards, in order:** + +1. **Reason gate.** `onPeerActivated` fires on non-handshake paths too. Only genuine re-handshake reasons carry a freshly-exchanged, trustworthy epoch. `healResetIncarnation` returns immediately unless `reason` is in the allow-list `HANDSHAKE_ACTIVATION_REASONS` (typed `ReadonlySet`): `initiate_accepted`, `accept_new`, `accept_pending`, `accept_rejected_override`, `accept_awaiting_approval`, `accept_awaiting_approval_fallback`, `approval_handshake`, `ensure_peered`. The two EXCLUDED reasons — `health_check_recovery` (reachability flip in `markPeerRecovered`) and `startup_bootstrap` (boot re-scan) — flip a peer to `active` **without** a handshake, so their baseline is stale (still equals the journaled `dead_epoch`). Without the gate they would hit the `deadEpoch === newEpoch` false-alarm branch and silently resolve the journal + clear the flags WITHOUT healing, permanently burying the bug. Gated out, they leave the journal fully intact for a later genuine re-handshake to heal. + +2. **Epoch comparison (false-positive guard).** For a gated-in reason, look up the UNRESOLVED `federation_reset_events` row for the origin (none → return): + - **`journal.dead_epoch === newEpoch`** — the re-peer confirmed the SAME incarnation (spurious/spoofed detection, or an admin re-peer to a never-reset live peer). **NO tombstone** — the user-level snapshot flags alone must never authorize destruction; only a confirmed epoch change does. Clears `federation_heal_pending` for the origin and resolves the journal (`new_epoch`, `resolved_at`). + - **`journal.dead_epoch !== newEpoch`** — a GENUINE new incarnation. Soft-tombstones the flagged **pure stubs** only, then clears their flags and resolves the journal. + +**Soft-tombstone (pure stubs only).** For every user that is `federation_heal_pending = 1` AND `password_hash = '!federation-replicated'` (pure S2S stub sentinel) AND matches the origin (`homeInstanceMatch`), calls `tombstoneUser(uid, { purgeContent: false })`. The `purgeContent: false` is **non-negotiable** — the default (`true`) irreversibly deletes this box's reactions and authored space messages, violating the invariant that a remote's reset never destroys our non-re-syncable content. The soft tombstone clears exactly the relationship rows that cause the bug (`friends`, `friend_requests`, `dm_members`, …) so stale friendships/DMs clear and re-adds work. Flags are then cleared **keyed by the stub id list** (not by re-querying the sentinel — `tombstoneUser` has already randomized `password_hash`). + +**Real federated accounts left intact.** A flagged user that is NOT a stub (`password_hash != '!federation-replicated'`) carries real, non-re-syncable local content. It is **never** auto-tombstoned — it stays `federation_heal_pending = 1` and fully intact for the Phase 2 quarantine/admin surface (design §6.3). + ### S2S Identity Deletion (`DELETE /api/federation/identity`) Allows a home instance to remove a user's replicated identity from a remote instance. diff --git a/packages/server/src/utils/federationPeerActivation.ts b/packages/server/src/utils/federationPeerActivation.ts index da131d12..07b767d3 100644 --- a/packages/server/src/utils/federationPeerActivation.ts +++ b/packages/server/src/utils/federationPeerActivation.ts @@ -4,6 +4,7 @@ import { and, eq } from 'drizzle-orm'; import { isFederationRelayEnabled } from './federationOutbox.js'; import { buildFederationHeaders, getOurOrigin } from './federationAuth.js'; import { generateSnowflake } from './snowflake.js'; +import { healResetIncarnation } from './federationReset.js'; import type { FederationRelayEvent } from '@backspace/shared'; export type PeerActivationReason = @@ -50,6 +51,25 @@ export async function onPeerActivated( const promise = (async () => { try { resetOutboxBackoff(peerId); + + // Instance-epoch self-heal. If this origin has an unresolved reset journal + // AND this is a genuine re-handshake activation (reason gate lives inside + // healResetIncarnation), heal the dead incarnation's stale stubs BEFORE the + // mutation-log re-sync below — so re-sync repopulates onto a clean slate + // (design §6.1). By this point the activation path has already (re)written + // peer_instance_id to the freshly-exchanged epoch. Runs OUTSIDE any + // transaction: tombstoneUser opens its own, and better-sqlite3 throws on a + // nested BEGIN. No-op on non-handshake reasons (health_check_recovery / + // startup_bootstrap) and when no reset is journaled. + const resetPeerRow = getDb() + .select({ origin: schema.federationPeers.origin, epoch: schema.federationPeers.peerInstanceId }) + .from(schema.federationPeers) + .where(eq(schema.federationPeers.id, peerId)) + .get(); + if (resetPeerRow?.epoch) { + healResetIncarnation(resetPeerRow.origin, resetPeerRow.epoch, reason); + } + await syncPeerMutationLog(peerId, reason); await fanoutOutboundSubscribers(peerId); diff --git a/packages/server/src/utils/federationReset.test.ts b/packages/server/src/utils/federationReset.test.ts index b0396668..6d7a149a 100644 --- a/packages/server/src/utils/federationReset.test.ts +++ b/packages/server/src/utils/federationReset.test.ts @@ -161,3 +161,119 @@ describe('markPeerReset — detection-only reset routing', () => { .where(eq(schema.users.id, 'stub-url')).get()!.federationHealPending).toBe(1); }); }); + +describe('healResetIncarnation — heal after authenticated re-peer', () => { + beforeEach(() => { + sqlite = new Database(':memory:'); + testDb = drizzle(sqlite, { schema }); + applyMigrations(sqlite); + vi.clearAllMocks(); + }); + + afterEach(() => { + sqlite.close(); + }); + + function seedJournal(deadEpoch: string): void { + testDb.insert(schema.federationResetEvents).values({ + origin: ORIGIN, deadEpoch, newEpoch: null, + detectedAt: Date.now(), resolvedAt: null, + stubCount: 1, orphanedAccountCount: 1, + }).run(); + } + + function flag(id: string): void { + testDb.update(schema.users).set({ federationHealPending: 1 }) + .where(eq(schema.users.id, id)).run(); + } + + it('genuine reset: soft-tombstones flagged stubs only, leaves real accounts flagged + intact, resolves journal', async () => { + seedPeer(); + seedJournal('E0'); + // A local native user to be the friendship counterpart. + testDb.insert(schema.users).values({ + id: 'local-1', username: 'alice', passwordHash: '$2b$10$localhash', + homeInstance: null, homeUserId: null, isDeleted: 0, createdAt: Date.now(), + }).run(); + // Flagged pure S2S stub with a friendship to the local user. + seedUser('stub-1', { passwordHash: STUB }); + flag('stub-1'); + testDb.insert(schema.friends).values({ + userId: 'stub-1', friendId: 'local-1', createdAt: Date.now(), + }).run(); + // Flagged REAL federated account (real bcrypt) — must survive untouched + still flagged. + seedUser('real-1', { passwordHash: '$2b$10$realbcrypthash' }); + flag('real-1'); + + const { healResetIncarnation } = await import('./federationReset.js'); + healResetIncarnation(ORIGIN, 'E1', 'initiate_accepted'); + + // Stub soft-tombstoned. + const stub = testDb.select().from(schema.users).where(eq(schema.users.id, 'stub-1')).get()!; + expect(stub.isDeleted).toBe(1); + expect(stub.username).toBe('!deleted:stub-1'); + // Its friendship row is gone → re-adds work again. + expect(testDb.select().from(schema.friends) + .where(eq(schema.friends.userId, 'stub-1')).all()).toHaveLength(0); + // Heal flag cleared on the healed stub. + expect(stub.federationHealPending).toBe(0); + + // Real account UNTOUCHED and STILL flagged (left for Phase 2 quarantine). + const real = testDb.select().from(schema.users).where(eq(schema.users.id, 'real-1')).get()!; + expect(real.isDeleted).toBe(0); + expect(real.username).toBe('real-1@peer.example'); + expect(real.federationHealPending).toBe(1); + + // Journal resolved with the freshly-handshaked epoch. + const journal = testDb.select().from(schema.federationResetEvents) + .where(eq(schema.federationResetEvents.origin, ORIGIN)).get()!; + expect(journal.resolvedAt).not.toBeNull(); + expect(journal.newEpoch).toBe('E1'); + }); + + it('false positive (re-peer confirmed same incarnation): NO tombstone, flags cleared, journal resolved', async () => { + seedPeer(); + seedJournal('E0'); + seedUser('stub-1', { passwordHash: STUB }); + flag('stub-1'); + + const { healResetIncarnation } = await import('./federationReset.js'); + healResetIncarnation(ORIGIN, 'E0', 'accept_new'); // dead_epoch === newEpoch → false alarm + + const stub = testDb.select().from(schema.users).where(eq(schema.users.id, 'stub-1')).get()!; + expect(stub.isDeleted).toBe(0); // NOT tombstoned + expect(stub.username).toBe('stub-1@peer.example'); + expect(stub.federationHealPending).toBe(0); // flag cleared + + const journal = testDb.select().from(schema.federationResetEvents) + .where(eq(schema.federationResetEvents.origin, ORIGIN)).get()!; + expect(journal.resolvedAt).not.toBeNull(); + expect(journal.newEpoch).toBe('E0'); + }); + + it('recovery/startup flip must NOT resolve the journal or clear flags (the critical guard)', async () => { + seedPeer(); + seedJournal('E0'); + seedUser('stub-1', { passwordHash: STUB }); + flag('stub-1'); + + const { healResetIncarnation } = await import('./federationReset.js'); + + // Both non-handshake reasons flip a peer to active with a STALE baseline + // (still E0). Without the reason gate they'd hit dead_epoch === newEpoch and + // silently resolve the journal WITHOUT healing → the bug permanently buried. + for (const reason of ['health_check_recovery', 'startup_bootstrap'] as const) { + healResetIncarnation(ORIGIN, 'E0', reason); + + const journal = testDb.select().from(schema.federationResetEvents) + .where(eq(schema.federationResetEvents.origin, ORIGIN)).get()!; + expect(journal.resolvedAt, `reason=${reason}`).toBeNull(); + expect(journal.newEpoch, `reason=${reason}`).toBeNull(); + + const stub = testDb.select().from(schema.users) + .where(eq(schema.users.id, 'stub-1')).get()!; + expect(stub.federationHealPending, `reason=${reason}`).toBe(1); + expect(stub.isDeleted, `reason=${reason}`).toBe(0); + } + }); +}); diff --git a/packages/server/src/utils/federationReset.ts b/packages/server/src/utils/federationReset.ts index f05fa706..92eea755 100644 --- a/packages/server/src/utils/federationReset.ts +++ b/packages/server/src/utils/federationReset.ts @@ -1,7 +1,9 @@ -import { and, eq, sql } from 'drizzle-orm'; +import { and, eq, inArray, isNull, sql } from 'drizzle-orm'; import { getDb, schema } from '../db/index.js'; import { extractDomain } from '../routes/federation.js'; import { connectionManager } from '../ws/handler.js'; +import { tombstoneUser } from './userDeletion.js'; +import type { PeerActivationReason } from './federationPeerActivation.js'; /** Pure-stub sentinel: a user replicated purely over S2S (no local credentials). */ const REPLICATED_STUB_SENTINEL = '!federation-replicated'; @@ -146,3 +148,138 @@ export function markPeerReset(peerId: string, origin: string, deadEpoch: string, connectionManager.sendToAdmins({ type: 'federation_peers_changed' as const }); connectionManager.sendToAdmins({ type: 'federation_peer_reset_detected' as const, origin }); } + +/** + * Activation reasons that involve a fresh, HMAC-authenticated handshake which + * (re)writes `federation_peers.peer_instance_id`. ONLY these reasons carry a + * freshly-exchanged epoch that can be trusted to confirm-or-refute a reset. + * + * The two EXCLUDED members of `PeerActivationReason` — `health_check_recovery` + * (a reachability flip in `markPeerRecovered`) and `startup_bootstrap` (a boot + * re-scan) — flip a peer to `active` WITHOUT any handshake, so the baseline they + * observe is STALE (still equal to the journaled `dead_epoch`). Letting the heal + * run on those paths would take the `deadEpoch === newEpoch` false-alarm branch + * and silently resolve the reset journal + clear the snapshot flags WITHOUT ever + * healing — permanently burying the bug. The reason gate below stops that: on a + * non-handshake activation the journal is left fully intact for a later genuine + * re-handshake to heal. + * + * Typed as `ReadonlySet` so a typo or a future + * union-member rename is caught by tsc, not at runtime. + */ +const HANDSHAKE_ACTIVATION_REASONS: ReadonlySet = new Set([ + 'initiate_accepted', + 'accept_new', + 'accept_pending', + 'accept_rejected_override', + 'accept_awaiting_approval', + 'accept_awaiting_approval_fallback', + 'approval_handshake', + 'ensure_peered', +]); + +/** + * The data self-heal, fired from `onPeerActivated` AFTER an authenticated + * re-peer (design §6). It is the counterpart to `markPeerReset`'s detection: + * detection snapshots + journals but never destroys; this heals once — and only + * once — the epoch change has been proven through a genuine handshake. + * + * Two mandatory guards, in order: + * + * 1. **Reason gate.** Returns immediately unless `reason` is a genuine + * handshake activation (see `HANDSHAKE_ACTIVATION_REASONS`). Reachability / + * startup flips carry a stale baseline and must leave the journal untouched. + * + * 2. **Epoch comparison (false-positive guard).** For a gated-in reason, look up + * the UNRESOLVED `federation_reset_events` row for the origin. If none → no + * outstanding reset → return. + * - `journal.deadEpoch === newEpoch`: the re-peer confirmed the SAME + * incarnation (a spurious/spoofed detection, or an admin re-peer to the + * never-reset live peer). **No tombstone** — the user-level snapshot flags + * alone must never drive destruction; only a confirmed epoch change + * authorizes it. Clear all heal flags for the origin and resolve the + * journal. + * - `journal.deadEpoch !== newEpoch`: a GENUINE new incarnation. Soft- + * tombstone the flagged PURE STUBS only, then clear their flags and resolve + * the journal. + * + * Real federated accounts (`federation_heal_pending = 1` but NOT a stub) carry + * non-re-syncable local content and are **never** auto-tombstoned; they stay + * flagged + intact for the Phase 2 quarantine/admin surface (design §6.3). + * + * **Transaction hazard:** `tombstoneUser` opens its OWN `db.transaction`, and + * better-sqlite3 throws on a nested `BEGIN`. `healResetIncarnation` therefore + * runs its `select`/`update` calls UNWRAPPED (never inside a transaction) and + * calls `tombstoneUser` per-stub outside any open transaction. The caller + * (`onPeerActivated`) must likewise not invoke this from within a transaction. + * + * @param origin The reset peer's origin (bare domain or full URL). + * @param newEpoch The peer's freshly-handshaked epoch (`peer_instance_id`). + * @param reason The activation reason that triggered this call. + */ +export function healResetIncarnation(origin: string, newEpoch: string, reason: PeerActivationReason): void { + // Guard 1 — reason gate: only a genuine re-handshake carries a trustworthy epoch. + if (!HANDSHAKE_ACTIVATION_REASONS.has(reason)) return; + + const db = getDb(); + + const journal = db + .select() + .from(schema.federationResetEvents) + .where(and( + eq(schema.federationResetEvents.origin, origin), + isNull(schema.federationResetEvents.resolvedAt), + )) + .get(); + if (!journal) return; // no outstanding reset for this origin + + // Guard 2 — epoch comparison (the false-positive guard). + if (journal.deadEpoch === newEpoch) { + // FALSE ALARM: re-peer confirmed the SAME incarnation. The snapshot flags + // alone must NEVER drive a tombstone — clear them and resolve, no deletion. + db.update(schema.users) + .set({ federationHealPending: 0 }) + .where(and(eq(schema.users.federationHealPending, 1), homeInstanceMatch(origin))) + .run(); + db.update(schema.federationResetEvents) + .set({ newEpoch, resolvedAt: Date.now() }) + .where(eq(schema.federationResetEvents.origin, origin)) + .run(); + return; + } + + // GENUINE reset: soft-tombstone the flagged PURE STUBS only. + const stubs = db + .select({ id: schema.users.id }) + .from(schema.users) + .where(and( + eq(schema.users.federationHealPending, 1), + eq(schema.users.passwordHash, REPLICATED_STUB_SENTINEL), + homeInstanceMatch(origin), + )) + .all(); + + // MANDATORY: purgeContent:false — a soft tombstone. The default (true) would + // irreversibly delete this box's reactions / space messages, violating the + // §1 invariant that a remote's reset never destroys our non-re-syncable + // content. Each call opens its own transaction, so this loop stays UNWRAPPED. + for (const stub of stubs) { + tombstoneUser(stub.id, { purgeContent: false }); + } + + // Clear the heal flag on exactly the stubs we healed, keyed by id. + // `tombstoneUser` has already randomized their `password_hash`, so re-querying + // by the stub sentinel would miss them — the id list is the reliable key. + // Real accounts keep `federation_heal_pending = 1` for Phase 2. + if (stubs.length > 0) { + db.update(schema.users) + .set({ federationHealPending: 0 }) + .where(inArray(schema.users.id, stubs.map((s) => s.id))) + .run(); + } + + db.update(schema.federationResetEvents) + .set({ newEpoch, resolvedAt: Date.now() }) + .where(eq(schema.federationResetEvents.origin, origin)) + .run(); +}