/** * One-time backfill: populate width/height for image embeds that lack them, * and create image embeds for old bare-image-URL messages that never got one * (these predate the `resolveEmbeds` call site in routes/messages.ts). * * Idempotent — re-running skips already-dim'd rows and already-embedded * messages via the WHERE clauses. * * Run from the running container: * docker exec backspace node dist/scripts/backfill-image-embed-dims.js */ import fs from 'fs'; import path from 'path'; import { initDatabase, getRawDb, schema } from '../db/index.js'; import { probeRemoteImageDimensions } from '../utils/embedResolver.js'; import { extractUrls } from '../utils/embedResolver.js'; import { classifyUrl } from '../utils/embedClassifier.js'; import { probeImageDimensions } from '../utils/thumbnail.js'; import { generateSnowflake } from '../utils/snowflake.js'; import { config } from '../config.js'; const CONCURRENCY = 4; interface PendingProbe { kind: 'update-embed' | 'create-embed-for-message' | 'create-embed-for-dm-message'; url: string; embedId?: string; messageId?: string; dmMessageId?: string; } async function withConcurrency(items: T[], n: number, worker: (item: T) => Promise): Promise { let i = 0; const runners = Array.from({ length: Math.min(n, items.length) }, async () => { while (true) { const idx = i++; if (idx >= items.length) return; const item = items[idx]; if (item === undefined) return; await worker(item); } }); await Promise.all(runners); } function isLikelyImageUrl(url: string): boolean { return classifyUrl(url).embedType === 'image'; } async function main(): Promise { // initDatabase() runs migrations + ensureDefaults + sets the snowflake // worker ID from instance_settings.worker_id. We rely on that for // generateSnowflake() below. initDatabase(); const db = getRawDb(); // ── Pass 1: image embeds missing dimensions ────────────────────────────── const nullDimEmbeds = db.prepare( `SELECT id, url FROM embeds WHERE embed_type = 'image' AND (width IS NULL OR height IS NULL)`, ).all() as Array<{ id: string; url: string }>; console.log(`[backfill] Found ${nullDimEmbeds.length} image embeds missing width/height`); // ── Pass 2: messages with bare image URLs but no embed row ─────────────── // SQLite has no REGEXP by default; use LIKE with the two known providers // plus a generic suffix match. We re-classify each candidate URL with // classifyUrl() before probing so non-image extensions are filtered out. const candidateMessages = db.prepare( `SELECT m.id, m.content FROM messages m LEFT JOIN embeds e ON e.message_id = m.id WHERE m.content IS NOT NULL AND ( m.content LIKE 'https://media.tenor.com/%' OR m.content LIKE 'https://static.klipy.com/%' OR m.content LIKE '%.gif' OR m.content LIKE '%.webp' ) AND e.id IS NULL`, ).all() as Array<{ id: string; content: string }>; console.log(`[backfill] Found ${candidateMessages.length} channel messages with image URLs and no embed`); const candidateDmMessages = db.prepare( `SELECT m.id, m.content FROM dm_messages m LEFT JOIN embeds e ON e.dm_message_id = m.id WHERE m.content IS NOT NULL AND ( m.content LIKE 'https://media.tenor.com/%' OR m.content LIKE 'https://static.klipy.com/%' OR m.content LIKE '%.gif' OR m.content LIKE '%.webp' ) AND e.id IS NULL`, ).all() as Array<{ id: string; content: string }>; console.log(`[backfill] Found ${candidateDmMessages.length} DM messages with image URLs and no embed`); // ── Build work queue ───────────────────────────────────────────────────── const work: PendingProbe[] = []; for (const row of nullDimEmbeds) { work.push({ kind: 'update-embed', url: row.url, embedId: row.id }); } for (const m of candidateMessages) { const urls = extractUrls(m.content); for (const url of urls) { if (isLikelyImageUrl(url)) { work.push({ kind: 'create-embed-for-message', url, messageId: m.id }); } } } for (const m of candidateDmMessages) { const urls = extractUrls(m.content); for (const url of urls) { if (isLikelyImageUrl(url)) { work.push({ kind: 'create-embed-for-dm-message', url, dmMessageId: m.id }); } } } // ── Pass 3: image attachments missing dimensions ───────────────────────── // Probe local files directly (the upload sits on the same filesystem). The // `failOn: 'none'` flag in `probeImageDimensions` handles animated GIFs // whose default sharp probe was failing on frame-data validation. const nullDimAttachments = db.prepare( `SELECT id, filename FROM attachments WHERE mimetype LIKE 'image/%' AND (width IS NULL OR height IS NULL)`, ).all() as Array<{ id: string; filename: string }>; console.log(`[backfill] Found ${nullDimAttachments.length} image attachments missing width/height`); const updateAttachment = db.prepare(`UPDATE attachments SET width = ?, height = ? WHERE id = ?`); let attachmentSuccesses = 0; let attachmentSkipped = 0; for (const a of nullDimAttachments) { const filepath = path.join(config.uploadDir, a.filename); if (!fs.existsSync(filepath)) { // Federated remote attachments live on the source instance, not here. // Their dims need to be backfilled from the source side, or via the // federation file-replication path. Skip silently. attachmentSkipped++; continue; } const dims = await probeImageDimensions(filepath); if (dims) { updateAttachment.run(dims.width, dims.height, a.id); attachmentSuccesses++; } } console.log(`[backfill] attachments updated=${attachmentSuccesses}, skipped (file not local)=${attachmentSkipped}`); // ── Pass 4: image embeds whose URL points at our local uploads ─────────── // SSRF correctly blocks the remote probe from hitting our own host. For // these specific embeds we have the file locally — extract the upload // filename from the URL path (`/api/uploads/`) and probe it // directly, sidestepping HTTP entirely. Federated `/api/uploads/...` URLs // on a different host won't have a local file and are skipped. const localUploadEmbeds = db.prepare( `SELECT id, url FROM embeds WHERE embed_type = 'image' AND (width IS NULL OR height IS NULL) AND url LIKE '%/api/uploads/%'`, ).all() as Array<{ id: string; url: string }>; console.log(`[backfill] Found ${localUploadEmbeds.length} image embeds pointing at /api/uploads`); const updateEmbedDims = db.prepare(`UPDATE embeds SET width = ?, height = ? WHERE id = ?`); let localEmbedSuccesses = 0; let localEmbedSkipped = 0; for (const e of localUploadEmbeds) { const match = e.url.match(/\/api\/uploads\/([^/?#]+)/); if (!match || !match[1]) { localEmbedSkipped++; continue; } const filepath = path.join(config.uploadDir, match[1]); if (!fs.existsSync(filepath)) { localEmbedSkipped++; continue; } const dims = await probeImageDimensions(filepath); if (dims) { updateEmbedDims.run(dims.width, dims.height, e.id); localEmbedSuccesses++; } } console.log(`[backfill] local-upload embeds updated=${localEmbedSuccesses}, skipped (no local file)=${localEmbedSkipped}`); console.log(`[backfill] ${work.length} remote probes to run (concurrency=${CONCURRENCY})`); // ── Execute ────────────────────────────────────────────────────────────── const updateEmbed = db.prepare( `UPDATE embeds SET width = ?, height = ? WHERE id = ?`, ); const insertEmbed = db.prepare( `INSERT INTO embeds (id, message_id, dm_message_id, url, embed_type, provider, title, description, image, embed_url, width, height, color, created_at) VALUES (?, ?, ?, ?, 'image', NULL, NULL, NULL, ?, NULL, ?, ?, NULL, ?)`, ); let successes = 0; let probeFailures = 0; let progress = 0; await withConcurrency(work, CONCURRENCY, async (item) => { const dims = await probeRemoteImageDimensions(item.url); progress++; if (progress % 25 === 0) { console.log(`[backfill] progress ${progress}/${work.length} (ok=${successes}, fail=${probeFailures})`); } if (!dims) { probeFailures++; return; } if (item.kind === 'update-embed' && item.embedId) { updateEmbed.run(dims.width, dims.height, item.embedId); successes++; } else if (item.kind === 'create-embed-for-message' && item.messageId) { insertEmbed.run( generateSnowflake(), item.messageId, null, item.url, item.url, dims.width, dims.height, Date.now(), ); successes++; } else if (item.kind === 'create-embed-for-dm-message' && item.dmMessageId) { insertEmbed.run( generateSnowflake(), null, item.dmMessageId, item.url, item.url, dims.width, dims.height, Date.now(), ); successes++; } }); console.log(`[backfill] done. successes=${successes}, probe-failures=${probeFailures}, total=${work.length}`); // Silence the schema import linter — it's intentionally available for // future per-table tweaks without re-importing. void schema; process.exit(0); } main().catch((err) => { console.error('[backfill] fatal:', err); process.exit(1); });