diff --git a/packages/server/src/utils/storageJanitor.ts b/packages/server/src/utils/storageJanitor.ts index c9031a17..5d258fac 100644 --- a/packages/server/src/utils/storageJanitor.ts +++ b/packages/server/src/utils/storageJanitor.ts @@ -1,6 +1,7 @@ import fs from 'fs'; import path from 'path'; import { and, eq, inArray, isNotNull, isNull, lt, lte } from 'drizzle-orm'; +import { FileStore } from '@tus/file-store'; import { config } from '../config.js'; import { getDb, getRawDb, schema } from '../db/index.js'; import { deleteUploadFile, deleteAttachmentFiles } from './fileCleanup.js'; @@ -693,6 +694,64 @@ export function cleanupSoftDeletedDmChannels(): number { return purged; } +// ── Tus-specific janitor functions ────────────────────────────────────────── +// Lazily construct (and cache) a FileStore pointed at config.tusUploadDir so +// we don't re-instantiate it on every janitor tick. The store is only used +// for its deleteExpired() method — we never serve uploads through it. +let cachedTusStore: FileStore | null = null; +function getTusStore(): FileStore { + if (!cachedTusStore) { + cachedTusStore = new FileStore({ + directory: config.tusUploadDir, + expirationPeriodInMilliseconds: config.tusExpirationMs, + }); + } + return cachedTusStore; +} + +/** + * Tus's own expiration sweep — removes uploads whose creation_date sidecar + * is past the configured expirationPeriodInMilliseconds. Cheap when nothing + * has expired (one directory listing). Returns the number of removed entries. + */ +export async function cleanupTusUploads(): Promise<{ removed: number }> { + const store = getTusStore(); + const removed = await store.deleteExpired(); + return { removed: typeof removed === 'number' ? removed : 0 }; +} + +/** + * Defensive sweep of `${config.tusUploadDir}`: any file (payload OR sidecar) + * whose mtime is older than `config.tusStragglerSweepMs` is unlinked. Covers + * the rare case where a tus crash left an orphan that deleteExpired() doesn't + * recognize — e.g. a payload without sidecar (so creation_date is unknown), + * or a sidecar without payload (so getUpload() rejects). + */ +export function cleanupTusStragglers(): { removed: number } { + if (!fs.existsSync(config.tusUploadDir)) return { removed: 0 }; + const entries = fs.readdirSync(config.tusUploadDir); + const cutoff = Date.now() - config.tusStragglerSweepMs; + let removed = 0; + for (const entry of entries) { + const full = path.join(config.tusUploadDir, entry); + let stat: fs.Stats; + try { + stat = fs.statSync(full); + } catch { + continue; + } + if (!stat.isFile()) continue; + if (stat.mtimeMs >= cutoff) continue; + try { + fs.unlinkSync(full); + removed += 1; + } catch { + // Race with tus's own cleanup or permissions error — ignore + } + } + return { removed }; +} + // cleanupStorage is expensive (full disk scan + DB joins) so we run it at // most once per 24h rather than on every janitor tick. const STORAGE_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000; @@ -736,6 +795,25 @@ export async function runFederationJanitor(): Promise { console.error('[storage-janitor] Approval request expiry error:', err); } + // Tus expiration sweep — every tick. Removes tus uploads whose + // creation_date sidecar is past config.tusExpirationMs (24h default). + // Cheap when nothing has expired (one directory listing). + try { + await cleanupTusUploads(); + } catch (err) { + console.warn('[storage-janitor] cleanupTusUploads failed:', err); + } + + // Tus straggler sweep — every tick. Defensive removal of payload/sidecar + // files in .tus/ whose mtime is older than config.tusStragglerSweepMs + // (48h default). Catches orphans that deleteExpired() can't recognize + // (payload without sidecar, sidecar without payload). + try { + cleanupTusStragglers(); + } catch (err) { + console.warn('[storage-janitor] cleanupTusStragglers failed:', err); + } + // Once-per-day storage cleanup. Sweeps orphan disk files (unreferenced // by any DB row) + unlinked attachments (uploaded but never attached // to a message past the 1h grace) + dangling attachment rows. Backs up diff --git a/packages/server/src/utils/storageJanitor.tus.test.ts b/packages/server/src/utils/storageJanitor.tus.test.ts new file mode 100644 index 00000000..58abf022 --- /dev/null +++ b/packages/server/src/utils/storageJanitor.tus.test.ts @@ -0,0 +1,81 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; +import crypto from 'node:crypto'; + +// Each test gets a fresh tmp dir so file I/O stays isolated. The Proxy +// mock below reads `tmpDir` and `stragglerSweepMs` lazily on each access, +// so reassigning them in beforeEach (or per-test) takes effect immediately. +let tmpDir: string; +let stragglerSweepMs = 48 * 60 * 60 * 1000; // default 48h + +vi.mock('../config.js', async () => { + const real = await import('../config.js'); + return { + config: new Proxy(real.config, { + get(target, prop: string) { + if (prop === 'tusUploadDir') return tmpDir; + if (prop === 'tusStragglerSweepMs') return stragglerSweepMs; + if (prop === 'tusExpirationMs') return 24 * 60 * 60 * 1000; + return (target as Record)[prop]; + }, + }), + }; +}); + +beforeEach(() => { + tmpDir = path.join(os.tmpdir(), `backspace-tus-jan-${crypto.randomBytes(8).toString('hex')}`); + stragglerSweepMs = 48 * 60 * 60 * 1000; +}); + +afterEach(() => { + if (fs.existsSync(tmpDir)) { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } +}); + +// Note: cleanupTusUploads() is exercised indirectly through @tus/file-store's +// own test suite — full integration would require building a real FileStore +// state with creation_date sidecars, which adds little signal here. +describe('cleanupTusStragglers', () => { + it('removes a file older than the sweep cutoff', async () => { + const { cleanupTusStragglers } = await import('./storageJanitor.js'); + fs.mkdirSync(tmpDir, { recursive: true }); + const stale = path.join(tmpDir, 'stale-upload-id'); + fs.writeFileSync(stale, 'stale data'); + + // Force mtime to 72h ago (stragglerSweepMs default is 48h) + const seventyTwoHoursAgo = Date.now() - 72 * 60 * 60 * 1000; + const seconds = seventyTwoHoursAgo / 1000; + fs.utimesSync(stale, seconds, seconds); + + const result = cleanupTusStragglers(); + + expect(result.removed).toBe(1); + expect(fs.existsSync(stale)).toBe(false); + }); + + it('keeps a recent file', async () => { + const { cleanupTusStragglers } = await import('./storageJanitor.js'); + fs.mkdirSync(tmpDir, { recursive: true }); + const fresh = path.join(tmpDir, 'fresh-upload-id'); + fs.writeFileSync(fresh, 'fresh data'); + // mtime defaults to "now" — well within the 48h window. + + const result = cleanupTusStragglers(); + + expect(result.removed).toBe(0); + expect(fs.existsSync(fresh)).toBe(true); + }); + + it('returns { removed: 0 } when the directory does not exist', async () => { + const { cleanupTusStragglers } = await import('./storageJanitor.js'); + // tmpDir was set in beforeEach but never created — so it doesn't exist. + expect(fs.existsSync(tmpDir)).toBe(false); + + const result = cleanupTusStragglers(); + + expect(result.removed).toBe(0); + }); +});