Merge branch 'feat/migration-squash-phase-2'
Squashes drizzle migrations 0000-0004 into a single baseline (0000_lethal_wildside.sql) and deletes the three migration adapter functions that existed only because of pre-squash intermediate states: baselineExistingInstall, healInitialSchemaDrift, healRenamedColumns. __drizzle_migrations surgery completed and verified live on Pi and VM before this merge. Both instances boot cleanly against the single baseline; cross-instance DM delivery verified. Closes backlog #31 Phase 2.
This commit is contained in:
@@ -1,7 +1,7 @@
|
|||||||
# Database Schema Reference
|
# Database Schema Reference
|
||||||
|
|
||||||
Source of truth: `packages/server/src/db/schema.ts` (Drizzle ORM)
|
Source of truth: `packages/server/src/db/schema.ts` (Drizzle ORM)
|
||||||
Migrations: `packages/server/src/db/migrate.ts` (runs on startup via `runMigrations()`)
|
Migrations: drizzle-kit generates SQL from `schema.ts` (`pnpm db:generate` from `packages/server/`). On startup, `initDatabase()` runs `drizzle.migrate()` against `packages/server/drizzle/`, then `ensureDefaults()` (settings row, Snowflake worker ID, first-admin promotion). Migration history was squashed to a single baseline on 2026-04-24 (backlog #31 Phase 2).
|
||||||
Engine: SQLite via `better-sqlite3`
|
Engine: SQLite via `better-sqlite3`
|
||||||
IDs: Snowflake text, permissions: bigint decimal strings
|
IDs: Snowflake text, permissions: bigint decimal strings
|
||||||
|
|
||||||
@@ -344,8 +344,6 @@ PK: (spaceId, userId, restrictionType)
|
|||||||
| autoAcceptPeering | integer NOT NULL | 1 | When 0, `peer/accept` rejects unsolicited requests with 403 |
|
| autoAcceptPeering | integer NOT NULL | 1 | When 0, `peer/accept` rejects unsolicited requests with 403 |
|
||||||
| updatedAt | integer NOT NULL | | |
|
| updatedAt | integer NOT NULL | | |
|
||||||
|
|
||||||
Migration flags (internal): `voice_bit_migrated`, `profile_attachments_cleaned`, `thumbnails_backfilled`, `media_dimensions_backfilled`, `legacy_dm_sync_done`
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Federation Tables
|
## Federation Tables
|
||||||
|
|||||||
+13
-1
@@ -188,7 +188,8 @@ CREATE TABLE `federation_peers` (
|
|||||||
`status` text DEFAULT 'active' NOT NULL,
|
`status` text DEFAULT 'active' NOT NULL,
|
||||||
`last_seen_at` integer,
|
`last_seen_at` integer,
|
||||||
`last_failure_at` integer,
|
`last_failure_at` integer,
|
||||||
`consecutive_failures` integer DEFAULT 0,
|
`consecutive_failures` integer DEFAULT 0 NOT NULL,
|
||||||
|
`consecutive_auth_failures` integer DEFAULT 0 NOT NULL,
|
||||||
`last_synced_at` integer DEFAULT 0,
|
`last_synced_at` integer DEFAULT 0,
|
||||||
`remote_max_upload_size` integer,
|
`remote_max_upload_size` integer,
|
||||||
`nonce_supported` integer DEFAULT 0 NOT NULL,
|
`nonce_supported` integer DEFAULT 0 NOT NULL,
|
||||||
@@ -238,6 +239,7 @@ CREATE TABLE `instance_settings` (
|
|||||||
`federation_relay_enabled` integer DEFAULT 1 NOT NULL,
|
`federation_relay_enabled` integer DEFAULT 1 NOT NULL,
|
||||||
`federation_relay_ttl_days` integer DEFAULT 30 NOT NULL,
|
`federation_relay_ttl_days` integer DEFAULT 30 NOT NULL,
|
||||||
`default_auto_rotate_interval_days` integer DEFAULT 90 NOT NULL,
|
`default_auto_rotate_interval_days` integer DEFAULT 90 NOT NULL,
|
||||||
|
`auto_accept_peering` integer DEFAULT 1 NOT NULL,
|
||||||
`updated_at` integer NOT NULL
|
`updated_at` integer NOT NULL
|
||||||
);
|
);
|
||||||
--> statement-breakpoint
|
--> statement-breakpoint
|
||||||
@@ -278,6 +280,15 @@ CREATE TABLE `messages` (
|
|||||||
FOREIGN KEY (`reply_to_id`) REFERENCES `messages`(`id`) ON UPDATE no action ON DELETE set null
|
FOREIGN KEY (`reply_to_id`) REFERENCES `messages`(`id`) ON UPDATE no action ON DELETE set null
|
||||||
);
|
);
|
||||||
--> statement-breakpoint
|
--> statement-breakpoint
|
||||||
|
CREATE TABLE `peer_approval_requests` (
|
||||||
|
`id` text PRIMARY KEY NOT NULL,
|
||||||
|
`origin` text NOT NULL,
|
||||||
|
`instance_name` text,
|
||||||
|
`hmac_secret` text NOT NULL,
|
||||||
|
`requested_at` integer NOT NULL,
|
||||||
|
`expires_at` integer NOT NULL
|
||||||
|
);
|
||||||
|
--> statement-breakpoint
|
||||||
CREATE TABLE `reactions` (
|
CREATE TABLE `reactions` (
|
||||||
`id` text PRIMARY KEY NOT NULL,
|
`id` text PRIMARY KEY NOT NULL,
|
||||||
`message_id` text NOT NULL,
|
`message_id` text NOT NULL,
|
||||||
@@ -436,6 +447,7 @@ CREATE INDEX `idx_join_requests_space_id_status` ON `join_requests` (`space_id`,
|
|||||||
CREATE INDEX `idx_member_roles_user_id_space_id` ON `member_roles` (`user_id`,`space_id`);--> statement-breakpoint
|
CREATE INDEX `idx_member_roles_user_id_space_id` ON `member_roles` (`user_id`,`space_id`);--> statement-breakpoint
|
||||||
CREATE INDEX `idx_messages_channel_id` ON `messages` (`channel_id`);--> statement-breakpoint
|
CREATE INDEX `idx_messages_channel_id` ON `messages` (`channel_id`);--> statement-breakpoint
|
||||||
CREATE INDEX `idx_messages_user_id` ON `messages` (`user_id`);--> statement-breakpoint
|
CREATE INDEX `idx_messages_user_id` ON `messages` (`user_id`);--> statement-breakpoint
|
||||||
|
CREATE UNIQUE INDEX `peer_approval_requests_origin_unique` ON `peer_approval_requests` (`origin`);--> statement-breakpoint
|
||||||
CREATE INDEX `idx_reactions_message_id` ON `reactions` (`message_id`);--> statement-breakpoint
|
CREATE INDEX `idx_reactions_message_id` ON `reactions` (`message_id`);--> statement-breakpoint
|
||||||
CREATE INDEX `idx_read_states_user_id` ON `read_states` (`user_id`);--> statement-breakpoint
|
CREATE INDEX `idx_read_states_user_id` ON `read_states` (`user_id`);--> statement-breakpoint
|
||||||
CREATE INDEX `idx_roles_space_id` ON `roles` (`space_id`);--> statement-breakpoint
|
CREATE INDEX `idx_roles_space_id` ON `roles` (`space_id`);--> statement-breakpoint
|
||||||
@@ -1 +0,0 @@
|
|||||||
ALTER TABLE `instance_settings` ADD `auto_accept_peering` integer DEFAULT 1 NOT NULL;
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
CREATE TABLE `peer_approval_requests` (
|
|
||||||
`id` text PRIMARY KEY NOT NULL,
|
|
||||||
`origin` text NOT NULL,
|
|
||||||
`instance_name` text,
|
|
||||||
`hmac_secret` text NOT NULL,
|
|
||||||
`requested_at` integer NOT NULL,
|
|
||||||
`expires_at` integer NOT NULL
|
|
||||||
);
|
|
||||||
--> statement-breakpoint
|
|
||||||
CREATE UNIQUE INDEX `peer_approval_requests_origin_unique` ON `peer_approval_requests` (`origin`);
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
ALTER TABLE `federation_peers` ADD `consecutive_auth_failures` integer DEFAULT 0 NOT NULL;
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
PRAGMA defer_foreign_keys=ON;
|
|
||||||
--> statement-breakpoint
|
|
||||||
CREATE TABLE `__new_federation_peers` (
|
|
||||||
`id` text PRIMARY KEY NOT NULL,
|
|
||||||
`origin` text NOT NULL,
|
|
||||||
`instance_name` text,
|
|
||||||
`hmac_secret` text NOT NULL,
|
|
||||||
`status` text DEFAULT 'active' NOT NULL,
|
|
||||||
`last_seen_at` integer,
|
|
||||||
`last_failure_at` integer,
|
|
||||||
`consecutive_failures` integer DEFAULT 0 NOT NULL,
|
|
||||||
`consecutive_auth_failures` integer DEFAULT 0 NOT NULL,
|
|
||||||
`last_synced_at` integer DEFAULT 0,
|
|
||||||
`remote_max_upload_size` integer,
|
|
||||||
`nonce_supported` integer DEFAULT 0 NOT NULL,
|
|
||||||
`pending_hmac_secret` text,
|
|
||||||
`secret_rotation_at` integer,
|
|
||||||
`secret_rotated_at` integer,
|
|
||||||
`auto_rotate_interval_days` integer DEFAULT 90 NOT NULL,
|
|
||||||
`created_at` integer NOT NULL
|
|
||||||
);
|
|
||||||
--> statement-breakpoint
|
|
||||||
INSERT INTO `__new_federation_peers`("id", "origin", "instance_name", "hmac_secret", "status", "last_seen_at", "last_failure_at", "consecutive_failures", "consecutive_auth_failures", "last_synced_at", "remote_max_upload_size", "nonce_supported", "pending_hmac_secret", "secret_rotation_at", "secret_rotated_at", "auto_rotate_interval_days", "created_at") SELECT "id", "origin", "instance_name", "hmac_secret", "status", "last_seen_at", "last_failure_at", COALESCE("consecutive_failures", 0), "consecutive_auth_failures", "last_synced_at", "remote_max_upload_size", "nonce_supported", "pending_hmac_secret", "secret_rotation_at", "secret_rotated_at", "auto_rotate_interval_days", "created_at" FROM `federation_peers`;
|
|
||||||
--> statement-breakpoint
|
|
||||||
DROP TABLE `federation_peers`;
|
|
||||||
--> statement-breakpoint
|
|
||||||
ALTER TABLE `__new_federation_peers` RENAME TO `federation_peers`;
|
|
||||||
--> statement-breakpoint
|
|
||||||
CREATE UNIQUE INDEX `federation_peers_origin_unique` ON `federation_peers` (`origin`);
|
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"version": "6",
|
"version": "6",
|
||||||
"dialect": "sqlite",
|
"dialect": "sqlite",
|
||||||
"id": "092e4ee0-51c4-4332-890c-fa011657a9a5",
|
"id": "b5b7d74f-4158-4798-9f01-2cbab126f9da",
|
||||||
"prevId": "00000000-0000-0000-0000-000000000000",
|
"prevId": "00000000-0000-0000-0000-000000000000",
|
||||||
"tables": {
|
"tables": {
|
||||||
"attachments": {
|
"attachments": {
|
||||||
@@ -1456,7 +1456,15 @@
|
|||||||
"name": "consecutive_failures",
|
"name": "consecutive_failures",
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"primaryKey": false,
|
"primaryKey": false,
|
||||||
"notNull": false,
|
"notNull": true,
|
||||||
|
"autoincrement": false,
|
||||||
|
"default": 0
|
||||||
|
},
|
||||||
|
"consecutive_auth_failures": {
|
||||||
|
"name": "consecutive_auth_failures",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
"autoincrement": false,
|
"autoincrement": false,
|
||||||
"default": 0
|
"default": 0
|
||||||
},
|
},
|
||||||
@@ -1850,6 +1858,14 @@
|
|||||||
"autoincrement": false,
|
"autoincrement": false,
|
||||||
"default": 90
|
"default": 90
|
||||||
},
|
},
|
||||||
|
"auto_accept_peering": {
|
||||||
|
"name": "auto_accept_peering",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false,
|
||||||
|
"default": 1
|
||||||
|
},
|
||||||
"updated_at": {
|
"updated_at": {
|
||||||
"name": "updated_at",
|
"name": "updated_at",
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
@@ -2179,6 +2195,65 @@
|
|||||||
"compositePrimaryKeys": {},
|
"compositePrimaryKeys": {},
|
||||||
"uniqueConstraints": {}
|
"uniqueConstraints": {}
|
||||||
},
|
},
|
||||||
|
"peer_approval_requests": {
|
||||||
|
"name": "peer_approval_requests",
|
||||||
|
"columns": {
|
||||||
|
"id": {
|
||||||
|
"name": "id",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": true,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"origin": {
|
||||||
|
"name": "origin",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"instance_name": {
|
||||||
|
"name": "instance_name",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": false,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"hmac_secret": {
|
||||||
|
"name": "hmac_secret",
|
||||||
|
"type": "text",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"requested_at": {
|
||||||
|
"name": "requested_at",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
},
|
||||||
|
"expires_at": {
|
||||||
|
"name": "expires_at",
|
||||||
|
"type": "integer",
|
||||||
|
"primaryKey": false,
|
||||||
|
"notNull": true,
|
||||||
|
"autoincrement": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"indexes": {
|
||||||
|
"peer_approval_requests_origin_unique": {
|
||||||
|
"name": "peer_approval_requests_origin_unique",
|
||||||
|
"columns": [
|
||||||
|
"origin"
|
||||||
|
],
|
||||||
|
"isUnique": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"foreignKeys": {},
|
||||||
|
"compositePrimaryKeys": {},
|
||||||
|
"uniqueConstraints": {}
|
||||||
|
},
|
||||||
"reactions": {
|
"reactions": {
|
||||||
"name": "reactions",
|
"name": "reactions",
|
||||||
"columns": {
|
"columns": {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -5,36 +5,8 @@
|
|||||||
{
|
{
|
||||||
"idx": 0,
|
"idx": 0,
|
||||||
"version": "6",
|
"version": "6",
|
||||||
"when": 1775687181033,
|
"when": 1777065939127,
|
||||||
"tag": "0000_initial",
|
"tag": "0000_lethal_wildside",
|
||||||
"breakpoints": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"idx": 1,
|
|
||||||
"version": "6",
|
|
||||||
"when": 1775734599940,
|
|
||||||
"tag": "0001_clear_earthquake",
|
|
||||||
"breakpoints": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"idx": 2,
|
|
||||||
"version": "6",
|
|
||||||
"when": 1776689384005,
|
|
||||||
"tag": "0002_peer_approval_requests",
|
|
||||||
"breakpoints": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"idx": 3,
|
|
||||||
"version": "6",
|
|
||||||
"when": 1776789567488,
|
|
||||||
"tag": "0003_classy_loki",
|
|
||||||
"breakpoints": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"idx": 4,
|
|
||||||
"version": "6",
|
|
||||||
"when": 1776802580650,
|
|
||||||
"tag": "0004_cooing_black_knight",
|
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { drizzle } from 'drizzle-orm/better-sqlite3';
|
|||||||
import { migrate } from 'drizzle-orm/better-sqlite3/migrator';
|
import { migrate } from 'drizzle-orm/better-sqlite3/migrator';
|
||||||
import { config } from '../config.js';
|
import { config } from '../config.js';
|
||||||
import * as schema from './schema.js';
|
import * as schema from './schema.js';
|
||||||
import { baselineExistingInstall, healInitialSchemaDrift, healRenamedColumns, ensureDefaults } from './migrate.js';
|
import { ensureDefaults } from './migrate.js';
|
||||||
import { setWorkerId } from '../utils/snowflake.js';
|
import { setWorkerId } from '../utils/snowflake.js';
|
||||||
import { mkdirSync } from 'fs';
|
import { mkdirSync } from 'fs';
|
||||||
import { dirname, resolve } from 'path';
|
import { dirname, resolve } from 'path';
|
||||||
@@ -24,24 +24,6 @@ export function initDatabase() {
|
|||||||
sqlite.pragma('journal_mode = WAL');
|
sqlite.pragma('journal_mode = WAL');
|
||||||
sqlite.pragma('foreign_keys = ON');
|
sqlite.pragma('foreign_keys = ON');
|
||||||
|
|
||||||
// Baseline existing installs before Drizzle migrate() runs —
|
|
||||||
// marks the initial migration as applied so it doesn't try to
|
|
||||||
// CREATE TABLE on a database that already has tables.
|
|
||||||
baselineExistingInstall(sqlite);
|
|
||||||
|
|
||||||
// Reconcile any columns present in the 0000 baseline but missing
|
|
||||||
// from the physical schema. Fixes drift between the pre-drizzle
|
|
||||||
// manual migration system and drizzle-kit's assumed baseline —
|
|
||||||
// a no-op on fresh installs and correctly-migrated DBs.
|
|
||||||
healInitialSchemaDrift(sqlite);
|
|
||||||
|
|
||||||
// Second-pass heal for rename/removal drift that the ADD-only pass
|
|
||||||
// can't resolve: for tables whose column set still doesn't match
|
|
||||||
// the 0000 snapshot and which are empty, DROP and rebuild from
|
|
||||||
// 0000_initial.sql. Non-empty tables are skipped with a warning.
|
|
||||||
// No-op on correctly-migrated DBs.
|
|
||||||
healRenamedColumns(sqlite);
|
|
||||||
|
|
||||||
// Apply any pending Drizzle migrations
|
// Apply any pending Drizzle migrations
|
||||||
const migrationsFolder = resolve(__dirname, '../../drizzle');
|
const migrationsFolder = resolve(__dirname, '../../drizzle');
|
||||||
const db = drizzle(sqlite, { schema });
|
const db = drizzle(sqlite, { schema });
|
||||||
|
|||||||
@@ -1,353 +1,5 @@
|
|||||||
import Database from 'better-sqlite3';
|
import Database from 'better-sqlite3';
|
||||||
import crypto from 'crypto';
|
import crypto from 'crypto';
|
||||||
import fs from 'fs';
|
|
||||||
import path from 'path';
|
|
||||||
import { fileURLToPath } from 'url';
|
|
||||||
|
|
||||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Baseline an existing install so Drizzle's migrate() skips the initial
|
|
||||||
* migration (tables already exist). Must be called BEFORE migrate().
|
|
||||||
*
|
|
||||||
* Detects existing installs by checking: users table exists but
|
|
||||||
* __drizzle_migrations table does not.
|
|
||||||
*
|
|
||||||
* Drizzle's __drizzle_migrations table schema (verified SQLite DDL):
|
|
||||||
* "id" integer PRIMARY KEY AUTOINCREMENT NOT NULL
|
|
||||||
* "hash" text NOT NULL -- SHA-256 hex of the migration SQL file (raw UTF-8)
|
|
||||||
* "created_at" numeric -- journalEntry.when (ms timestamp from journal)
|
|
||||||
*
|
|
||||||
* Hash must match Drizzle's exactly: read SQL file as raw UTF-8 string,
|
|
||||||
* hash it with SHA-256. Line endings matter — don't normalize \r\n vs \n.
|
|
||||||
*/
|
|
||||||
export function baselineExistingInstall(db: Database.Database): void {
|
|
||||||
const hasUsers = db.prepare(
|
|
||||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='users'"
|
|
||||||
).get();
|
|
||||||
const hasJournal = db.prepare(
|
|
||||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='__drizzle_migrations'"
|
|
||||||
).get();
|
|
||||||
|
|
||||||
if (!hasUsers || hasJournal) return; // Fresh install or already baselined
|
|
||||||
|
|
||||||
console.log('[migrate] Existing install detected — baselining Drizzle migrations...');
|
|
||||||
|
|
||||||
// Read the journal to get the initial migration metadata
|
|
||||||
const migrationsFolder = path.resolve(__dirname, '../../drizzle');
|
|
||||||
const journalPath = path.join(migrationsFolder, 'meta', '_journal.json');
|
|
||||||
const journal = JSON.parse(fs.readFileSync(journalPath, 'utf-8'));
|
|
||||||
const initialEntry = journal.entries[0];
|
|
||||||
|
|
||||||
if (!initialEntry) {
|
|
||||||
throw new Error('No entries found in drizzle migration journal');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compute the hash the same way Drizzle does: SHA-256 of the SQL file content
|
|
||||||
const sqlPath = path.join(migrationsFolder, `${initialEntry.tag}.sql`);
|
|
||||||
const sqlContent = fs.readFileSync(sqlPath, 'utf-8');
|
|
||||||
const hash = crypto.createHash('sha256').update(sqlContent).digest('hex');
|
|
||||||
|
|
||||||
// Create the journal table matching Drizzle's exact SQLite DDL
|
|
||||||
db.exec(`
|
|
||||||
CREATE TABLE IF NOT EXISTS "__drizzle_migrations" (
|
|
||||||
"id" integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
|
||||||
"hash" text NOT NULL,
|
|
||||||
"created_at" numeric
|
|
||||||
)
|
|
||||||
`);
|
|
||||||
|
|
||||||
db.prepare(
|
|
||||||
'INSERT INTO __drizzle_migrations (hash, created_at) VALUES (?, ?)'
|
|
||||||
).run(hash, initialEntry.when);
|
|
||||||
|
|
||||||
console.log(`[migrate] Baselined initial migration: ${initialEntry.tag} (hash: ${hash.slice(0, 12)}...)`);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Heal column drift between an existing install and the 0000_initial
|
|
||||||
* schema baseline.
|
|
||||||
*
|
|
||||||
* The baseline in `baselineExistingInstall` assumes that any existing
|
|
||||||
* install's schema already matches 0000_initial.sql. That assumption is
|
|
||||||
* usually fine for installs created through the old pre-drizzle manual
|
|
||||||
* migration system — but a DB can fall behind if it skipped one of those
|
|
||||||
* manual ALTERs (e.g. a dev-env that was paused before the
|
|
||||||
* `remote_max_upload_size` migration in 1e4e71a landed). On such DBs,
|
|
||||||
* baseline marks 0000 as applied without the columns actually being
|
|
||||||
* there, and a later migration that recreates the table (e.g.
|
|
||||||
* 0004_cooing_black_knight) crashes trying to SELECT them.
|
|
||||||
*
|
|
||||||
* Walks every table defined in 0000_snapshot.json; for each that already
|
|
||||||
* exists in the DB, ADDs any columns the snapshot declares but the table
|
|
||||||
* is missing. Idempotent: on a correctly-migrated DB every column is
|
|
||||||
* already present and the loop is a no-op. Columns that SQLite's
|
|
||||||
* ALTER TABLE ADD COLUMN cannot express (PRIMARY KEY; NOT NULL without a
|
|
||||||
* default) are skipped with a warning rather than corrupting data.
|
|
||||||
*
|
|
||||||
* Only reconciles against the 0000 baseline — later migrations add their
|
|
||||||
* own columns through normal migration SQL and are handled by
|
|
||||||
* Drizzle's migrator.
|
|
||||||
*/
|
|
||||||
export function healInitialSchemaDrift(db: Database.Database): void {
|
|
||||||
const migrationsFolder = path.resolve(__dirname, '../../drizzle');
|
|
||||||
const snapshotPath = path.join(migrationsFolder, 'meta', '0000_snapshot.json');
|
|
||||||
const snapshot = JSON.parse(fs.readFileSync(snapshotPath, 'utf-8')) as {
|
|
||||||
tables: Record<string, {
|
|
||||||
name: string;
|
|
||||||
columns: Record<string, {
|
|
||||||
name: string;
|
|
||||||
type: string;
|
|
||||||
primaryKey: boolean;
|
|
||||||
notNull: boolean;
|
|
||||||
autoincrement: boolean;
|
|
||||||
default?: string | number;
|
|
||||||
}>;
|
|
||||||
}>;
|
|
||||||
};
|
|
||||||
|
|
||||||
for (const table of Object.values(snapshot.tables)) {
|
|
||||||
const exists = db.prepare(
|
|
||||||
"SELECT name FROM sqlite_master WHERE type='table' AND name=?"
|
|
||||||
).get(table.name);
|
|
||||||
if (!exists) continue;
|
|
||||||
|
|
||||||
const existingCols = new Set(
|
|
||||||
(db.prepare(`PRAGMA table_info("${table.name}")`).all() as { name: string }[])
|
|
||||||
.map(c => c.name)
|
|
||||||
);
|
|
||||||
|
|
||||||
for (const col of Object.values(table.columns)) {
|
|
||||||
if (existingCols.has(col.name)) continue;
|
|
||||||
|
|
||||||
if (col.primaryKey) {
|
|
||||||
console.warn(`[migrate] Skipping drift heal for ${table.name}.${col.name}: PRIMARY KEY cannot be added via ALTER TABLE`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (col.notNull && col.default === undefined) {
|
|
||||||
console.warn(`[migrate] Skipping drift heal for ${table.name}.${col.name}: NOT NULL with no default (would violate existing rows)`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const parts = [`"${col.name}"`, col.type];
|
|
||||||
if (col.notNull) parts.push('NOT NULL');
|
|
||||||
if (col.default !== undefined) parts.push(`DEFAULT ${col.default}`);
|
|
||||||
|
|
||||||
const sql = `ALTER TABLE "${table.name}" ADD COLUMN ${parts.join(' ')}`;
|
|
||||||
console.log(`[migrate] Healing schema drift: ${sql}`);
|
|
||||||
db.exec(sql);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Second-pass heal for pre-drizzle dev DBs whose column drift is not a
|
|
||||||
* pure add-column problem.
|
|
||||||
*
|
|
||||||
* `healInitialSchemaDrift` handles tables that are missing columns — it
|
|
||||||
* ALTERs them in. But a pre-drizzle dev DB can also carry *renamed*
|
|
||||||
* columns: e.g. `federation_outbox` on some old installs retains the
|
|
||||||
* pre-rename `message_id` / `dm_channel_id` columns instead of the
|
|
||||||
* current `entity_id` / `context_id`. The missing columns are NOT NULL
|
|
||||||
* without a default, so the earlier heal correctly skips them, and the
|
|
||||||
* outbox worker then fails every tick with `no such column:
|
|
||||||
* federation_outbox.context_id`.
|
|
||||||
*
|
|
||||||
* For tables whose physical column set is *missing* columns declared by
|
|
||||||
* the current-migration-state snapshot (the one matching the highest
|
|
||||||
* `__drizzle_migrations` entry — NOT the latest snapshot on disk) *and*
|
|
||||||
* which hold zero rows, this pass DROPs the table and rebuilds it from
|
|
||||||
* the snapshot's JSON (columns, defaults, foreign keys, composite PKs,
|
|
||||||
* unique constraints, indexes). Targeting the current-state snapshot
|
|
||||||
* — rather than the latest — is critical: rebuilding to a future
|
|
||||||
* snapshot would introduce columns that drizzle's migrator is about
|
|
||||||
* to add via ALTER TABLE ADD COLUMN, causing duplicate-column errors.
|
|
||||||
* Extra-only drift (leftover columns from pre-drizzle manual migrations
|
|
||||||
* that no current code reads) is left alone — the trigger is missing
|
|
||||||
* columns, because that's what breaks runtime queries.
|
|
||||||
*
|
|
||||||
* Non-empty tables are skipped with a warning — data preservation wins
|
|
||||||
* over heal, and this path should only ever be hit by a pre-drizzle dev
|
|
||||||
* DB that never exercised the affected tables in the first place.
|
|
||||||
*
|
|
||||||
* Idempotent: on correctly-migrated DBs the column sets already match,
|
|
||||||
* so the mismatch check short-circuits for every table and nothing is
|
|
||||||
* dropped. Production Pi+VM instances are unaffected.
|
|
||||||
*/
|
|
||||||
type SnapshotColumn = {
|
|
||||||
name: string;
|
|
||||||
type: string;
|
|
||||||
primaryKey: boolean;
|
|
||||||
notNull: boolean;
|
|
||||||
autoincrement: boolean;
|
|
||||||
default?: string | number;
|
|
||||||
};
|
|
||||||
|
|
||||||
type SnapshotTable = {
|
|
||||||
name: string;
|
|
||||||
columns: Record<string, SnapshotColumn>;
|
|
||||||
indexes: Record<string, { name: string; columns: string[]; isUnique: boolean }>;
|
|
||||||
foreignKeys: Record<string, {
|
|
||||||
name: string;
|
|
||||||
tableFrom: string;
|
|
||||||
tableTo: string;
|
|
||||||
columnsFrom: string[];
|
|
||||||
columnsTo: string[];
|
|
||||||
onDelete?: string;
|
|
||||||
onUpdate?: string;
|
|
||||||
}>;
|
|
||||||
compositePrimaryKeys: Record<string, { name?: string; columns: string[] }>;
|
|
||||||
uniqueConstraints: Record<string, { name?: string; columns: string[] }>;
|
|
||||||
};
|
|
||||||
|
|
||||||
type DrizzleSnapshot = { tables: Record<string, SnapshotTable> };
|
|
||||||
|
|
||||||
function loadSnapshotForCurrentMigrationState(
|
|
||||||
db: Database.Database,
|
|
||||||
migrationsFolder: string
|
|
||||||
): DrizzleSnapshot {
|
|
||||||
// The rebuild target must be the snapshot that reflects what drizzle
|
|
||||||
// has already applied — NOT the latest snapshot on disk. Rebuilding
|
|
||||||
// to a future snapshot would introduce columns that pending ALTER
|
|
||||||
// TABLE ADD COLUMN migrations are about to add, causing duplicate-
|
|
||||||
// column errors when drizzle's migrator runs right after.
|
|
||||||
//
|
|
||||||
// __drizzle_migrations rows store `created_at` = the journal entry's
|
|
||||||
// `when` timestamp. Pick the row with the highest id (most recently
|
|
||||||
// applied), match it to a journal entry by that timestamp, then walk
|
|
||||||
// backward to find the nearest existing snapshot (some idx values
|
|
||||||
// may lack a snapshot if the migration was hand-written outside of
|
|
||||||
// `drizzle-kit generate`, e.g. 0002 in this codebase).
|
|
||||||
const journalPath = path.join(migrationsFolder, 'meta', '_journal.json');
|
|
||||||
const journal = JSON.parse(fs.readFileSync(journalPath, 'utf-8')) as {
|
|
||||||
entries: { idx: number; tag: string; when: number }[];
|
|
||||||
};
|
|
||||||
|
|
||||||
const hasJournal = db.prepare(
|
|
||||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='__drizzle_migrations'"
|
|
||||||
).get();
|
|
||||||
if (!hasJournal) {
|
|
||||||
// No migrations recorded at all — caller is about to run them all
|
|
||||||
// from scratch via drizzle's migrate(). Nothing to rebuild against;
|
|
||||||
// return an empty snapshot so the heal loop is a no-op.
|
|
||||||
return { tables: {} };
|
|
||||||
}
|
|
||||||
|
|
||||||
const latest = db.prepare(
|
|
||||||
'SELECT created_at FROM __drizzle_migrations ORDER BY id DESC LIMIT 1'
|
|
||||||
).get() as { created_at: number } | undefined;
|
|
||||||
if (!latest) return { tables: {} };
|
|
||||||
|
|
||||||
const entry = journal.entries.find(e => e.when === latest.created_at);
|
|
||||||
if (!entry) {
|
|
||||||
throw new Error(
|
|
||||||
`__drizzle_migrations.created_at=${latest.created_at} has no matching journal entry`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const candidates = journal.entries
|
|
||||||
.filter(e => e.idx <= entry.idx)
|
|
||||||
.sort((a, b) => b.idx - a.idx);
|
|
||||||
for (const candidate of candidates) {
|
|
||||||
const idxPadded = String(candidate.idx).padStart(4, '0');
|
|
||||||
const snapshotPath = path.join(migrationsFolder, 'meta', `${idxPadded}_snapshot.json`);
|
|
||||||
if (fs.existsSync(snapshotPath)) {
|
|
||||||
return JSON.parse(fs.readFileSync(snapshotPath, 'utf-8')) as DrizzleSnapshot;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// No snapshot at or below current migration state — shouldn't happen
|
|
||||||
// in practice because 0000_snapshot.json always exists, but fall
|
|
||||||
// through to a no-op rather than guessing.
|
|
||||||
return { tables: {} };
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildCreateTableStatement(table: SnapshotTable): string {
|
|
||||||
const lines: string[] = [];
|
|
||||||
for (const col of Object.values(table.columns)) {
|
|
||||||
const parts = [`\`${col.name}\``, col.type];
|
|
||||||
if (col.primaryKey) parts.push('PRIMARY KEY');
|
|
||||||
if (col.notNull) parts.push('NOT NULL');
|
|
||||||
if (col.default !== undefined) parts.push(`DEFAULT ${col.default}`);
|
|
||||||
lines.push('\t' + parts.join(' '));
|
|
||||||
}
|
|
||||||
for (const cpk of Object.values(table.compositePrimaryKeys)) {
|
|
||||||
lines.push('\tPRIMARY KEY(' + cpk.columns.map(c => `\`${c}\``).join(', ') + ')');
|
|
||||||
}
|
|
||||||
for (const uc of Object.values(table.uniqueConstraints)) {
|
|
||||||
lines.push('\tUNIQUE(' + uc.columns.map(c => `\`${c}\``).join(', ') + ')');
|
|
||||||
}
|
|
||||||
for (const fk of Object.values(table.foreignKeys)) {
|
|
||||||
const from = fk.columnsFrom.map(c => `\`${c}\``).join(', ');
|
|
||||||
const to = fk.columnsTo.map(c => `\`${c}\``).join(', ');
|
|
||||||
const onUpdate = fk.onUpdate ? ` ON UPDATE ${fk.onUpdate}` : '';
|
|
||||||
const onDelete = fk.onDelete ? ` ON DELETE ${fk.onDelete}` : '';
|
|
||||||
lines.push(`\tFOREIGN KEY (${from}) REFERENCES \`${fk.tableTo}\`(${to})${onUpdate}${onDelete}`);
|
|
||||||
}
|
|
||||||
return `CREATE TABLE \`${table.name}\` (\n${lines.join(',\n')}\n)`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildIndexStatements(table: SnapshotTable): string[] {
|
|
||||||
return Object.values(table.indexes).map(idx => {
|
|
||||||
const unique = idx.isUnique ? 'UNIQUE ' : '';
|
|
||||||
const cols = idx.columns.map(c => `\`${c}\``).join(', ');
|
|
||||||
return `CREATE ${unique}INDEX \`${idx.name}\` ON \`${table.name}\` (${cols})`;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export function healRenamedColumns(db: Database.Database): void {
|
|
||||||
const migrationsFolder = path.resolve(__dirname, '../../drizzle');
|
|
||||||
const snapshot = loadSnapshotForCurrentMigrationState(db, migrationsFolder);
|
|
||||||
|
|
||||||
for (const table of Object.values(snapshot.tables)) {
|
|
||||||
const exists = db.prepare(
|
|
||||||
"SELECT name FROM sqlite_master WHERE type='table' AND name=?"
|
|
||||||
).get(table.name);
|
|
||||||
if (!exists) continue;
|
|
||||||
|
|
||||||
const snapshotCols = new Set(Object.keys(table.columns));
|
|
||||||
const physicalCols = new Set(
|
|
||||||
(db.prepare(`PRAGMA table_info("${table.name}")`).all() as { name: string }[])
|
|
||||||
.map(c => c.name)
|
|
||||||
);
|
|
||||||
|
|
||||||
const missingFromPhysical = [...snapshotCols].filter(c => !physicalCols.has(c));
|
|
||||||
const extraInPhysical = [...physicalCols].filter(c => !snapshotCols.has(c));
|
|
||||||
|
|
||||||
// Trigger the rebuild only when the physical table is *missing*
|
|
||||||
// columns the snapshot declares — that's the runtime-breaking
|
|
||||||
// case the app code relies on. Extra-only drift (leftover columns
|
|
||||||
// from pre-drizzle manual migrations that no current code reads)
|
|
||||||
// is left alone: dropping those would risk losing data the app
|
|
||||||
// doesn't care about but the operator might.
|
|
||||||
if (missingFromPhysical.length === 0) continue;
|
|
||||||
|
|
||||||
const rowCount = (db.prepare(
|
|
||||||
`SELECT COUNT(*) AS c FROM "${table.name}"`
|
|
||||||
).get() as { c: number }).c;
|
|
||||||
|
|
||||||
if (rowCount > 0) {
|
|
||||||
console.warn(
|
|
||||||
`[migrate] Column drift on ${table.name} (missing: [${missingFromPhysical.join(', ')}], extra: [${extraInPhysical.join(', ')}]) — skipping rebuild: table has ${rowCount} rows, manual data migration required`
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
`[migrate] Rebuilding empty table ${table.name} to match current-migration-state snapshot (missing: [${missingFromPhysical.join(', ')}], extra: [${extraInPhysical.join(', ')}])`
|
|
||||||
);
|
|
||||||
|
|
||||||
const createStmt = buildCreateTableStatement(table);
|
|
||||||
const indexStmts = buildIndexStatements(table);
|
|
||||||
|
|
||||||
// Wrap in a transaction so a partial rebuild rolls back cleanly.
|
|
||||||
const rebuild = db.transaction(() => {
|
|
||||||
db.exec(`DROP TABLE "${table.name}"`);
|
|
||||||
db.exec(createStmt);
|
|
||||||
for (const stmt of indexStmts) db.exec(stmt);
|
|
||||||
});
|
|
||||||
rebuild();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Ensure data invariants after schema migration. Idempotent — safe to run
|
* Ensure data invariants after schema migration. Idempotent — safe to run
|
||||||
|
|||||||
Reference in New Issue
Block a user