From e703e29f8ae6d4b5d38d8b92e965cf01fe32958a Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Thu, 23 Apr 2026 03:00:30 +0200 Subject: [PATCH] fix(server): heal 0000-baseline schema drift after baselining existing installs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit baselineExistingInstall marks 0000_initial as applied when it detects pre-existing tables, on the assumption the install's schema matches the 0000 baseline. That assumption is false for dev DBs created under the pre-drizzle manual migrate.ts system that skipped or never ran some of its idempotent ALTER TABLE steps — for example the b9e4c65 migration that added federation_peers.remote_max_upload_size. On such DBs, 0000 is marked done without the column actually existing, and a later migration that recreates the table (0004_cooing_black_knight) subsequently crashes with "no such column: remote_max_upload_size" while building its __new_federation_peers SELECT. Adds healInitialSchemaDrift(): walks every table in 0000_snapshot.json, and for each table that already exists, ADDs any columns the snapshot declares but the physical table is missing. Runs immediately after baselining, before drizzle's migrate() — so later migrations find the schema they expect. Columns that SQLite's ALTER TABLE ADD COLUMN can't safely express (PRIMARY KEY; NOT NULL without a default) are skipped with a warning rather than corrupting data. Idempotent: on fresh installs and correctly-migrated DBs every column is already present, so the loop is a no-op. Production Pi+VM instances are unaffected. Verification: local dev DB that previously crashed on 0004 now boots cleanly — federation_peers gained remote_max_upload_size, nonce_supported, pending_hmac_secret, secret_rotation_at, secret_rotated_at, and auto_rotate_interval_days; __drizzle_migrations advanced from 4 to 5 entries; server binds :3005. 110/110 server tests + 131/131 web tests still pass. Not covered: a deeper drift on federation_outbox / federation_mutation_log where the physical tables retain pre-rename column names (message_id / dm_channel_id) instead of the current entity_id / context_id. Heal skips those (NOT NULL without default) and the outbox worker emits SQLITE_ERROR ticks post-boot. Both tables are empty on affected dev DBs, but a clean fix requires DROP + RECREATE with index reinstatement which is out of #29's stated scope ("column existing"). Flagged for a follow-up. Closes backlog #29. --- packages/server/src/db/index.ts | 8 +++- packages/server/src/db/migrate.ts | 76 +++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/packages/server/src/db/index.ts b/packages/server/src/db/index.ts index 4299bf25..44840bba 100644 --- a/packages/server/src/db/index.ts +++ b/packages/server/src/db/index.ts @@ -3,7 +3,7 @@ import { drizzle } from 'drizzle-orm/better-sqlite3'; import { migrate } from 'drizzle-orm/better-sqlite3/migrator'; import { config } from '../config.js'; import * as schema from './schema.js'; -import { baselineExistingInstall, ensureDefaults } from './migrate.js'; +import { baselineExistingInstall, healInitialSchemaDrift, ensureDefaults } from './migrate.js'; import { setWorkerId } from '../utils/snowflake.js'; import { mkdirSync } from 'fs'; import { dirname, resolve } from 'path'; @@ -29,6 +29,12 @@ export function initDatabase() { // CREATE TABLE on a database that already has tables. baselineExistingInstall(sqlite); + // Reconcile any columns present in the 0000 baseline but missing + // from the physical schema. Fixes drift between the pre-drizzle + // manual migration system and drizzle-kit's assumed baseline — + // a no-op on fresh installs and correctly-migrated DBs. + healInitialSchemaDrift(sqlite); + // Apply any pending Drizzle migrations const migrationsFolder = resolve(__dirname, '../../drizzle'); const db = drizzle(sqlite, { schema }); diff --git a/packages/server/src/db/migrate.ts b/packages/server/src/db/migrate.ts index 1038b65c..52538401 100644 --- a/packages/server/src/db/migrate.ts +++ b/packages/server/src/db/migrate.ts @@ -64,6 +64,82 @@ export function baselineExistingInstall(db: Database.Database): void { console.log(`[migrate] Baselined initial migration: ${initialEntry.tag} (hash: ${hash.slice(0, 12)}...)`); } +/** + * Heal column drift between an existing install and the 0000_initial + * schema baseline. + * + * The baseline in `baselineExistingInstall` assumes that any existing + * install's schema already matches 0000_initial.sql. That assumption is + * usually fine for installs created through the old pre-drizzle manual + * migration system — but a DB can fall behind if it skipped one of those + * manual ALTERs (e.g. a dev-env that was paused before the + * `remote_max_upload_size` migration in 1e4e71a landed). On such DBs, + * baseline marks 0000 as applied without the columns actually being + * there, and a later migration that recreates the table (e.g. + * 0004_cooing_black_knight) crashes trying to SELECT them. + * + * Walks every table defined in 0000_snapshot.json; for each that already + * exists in the DB, ADDs any columns the snapshot declares but the table + * is missing. Idempotent: on a correctly-migrated DB every column is + * already present and the loop is a no-op. Columns that SQLite's + * ALTER TABLE ADD COLUMN cannot express (PRIMARY KEY; NOT NULL without a + * default) are skipped with a warning rather than corrupting data. + * + * Only reconciles against the 0000 baseline — later migrations add their + * own columns through normal migration SQL and are handled by + * Drizzle's migrator. + */ +export function healInitialSchemaDrift(db: Database.Database): void { + const migrationsFolder = path.resolve(__dirname, '../../drizzle'); + const snapshotPath = path.join(migrationsFolder, 'meta', '0000_snapshot.json'); + const snapshot = JSON.parse(fs.readFileSync(snapshotPath, 'utf-8')) as { + tables: Record; + }>; + }; + + for (const table of Object.values(snapshot.tables)) { + const exists = db.prepare( + "SELECT name FROM sqlite_master WHERE type='table' AND name=?" + ).get(table.name); + if (!exists) continue; + + const existingCols = new Set( + (db.prepare(`PRAGMA table_info("${table.name}")`).all() as { name: string }[]) + .map(c => c.name) + ); + + for (const col of Object.values(table.columns)) { + if (existingCols.has(col.name)) continue; + + if (col.primaryKey) { + console.warn(`[migrate] Skipping drift heal for ${table.name}.${col.name}: PRIMARY KEY cannot be added via ALTER TABLE`); + continue; + } + if (col.notNull && col.default === undefined) { + console.warn(`[migrate] Skipping drift heal for ${table.name}.${col.name}: NOT NULL with no default (would violate existing rows)`); + continue; + } + + const parts = [`"${col.name}"`, col.type]; + if (col.notNull) parts.push('NOT NULL'); + if (col.default !== undefined) parts.push(`DEFAULT ${col.default}`); + + const sql = `ALTER TABLE "${table.name}" ADD COLUMN ${parts.join(' ')}`; + console.log(`[migrate] Healing schema drift: ${sql}`); + db.exec(sql); + } + } +} + /** * Ensure data invariants after schema migration. Idempotent — safe to run * on every boot. Uses raw better-sqlite3 handle (not Drizzle ORM).