diff --git a/docs/systems/admin.md b/docs/systems/admin.md index 46d62734..8ee0d61e 100644 --- a/docs/systems/admin.md +++ b/docs/systems/admin.md @@ -204,8 +204,11 @@ GET /api/admin/storage/stats → StorageStats GET /api/admin/storage/orphans → { orphans: OrphanedFile[] } POST /api/admin/storage/cleanup { dryRun?: boolean } → CleanupResult POST /api/admin/storage/cleanup-media { maxAgeDays: number, dryRun?: boolean } → CleanupResult +POST /api/admin/storage/cleanup-tus { maxAgeHours?: number = 1, dryRun?: boolean = false } → CleanupResult ``` +The `cleanup-tus` route walks `.tus/`, deleting (or counting, if `dryRun`) any entry whose mtime is older than `maxAgeHours`. No DB rows are touched — `.tus/` is filesystem-only — so `deletedAttachmentRecords` in the response is always `0`. See `docs/systems/uploads.md` §Janitor for the full lifecycle (immediate-DELETE on cancel/discard, automatic 24 h `cleanupTusUploads`, 48 h defensive `cleanupTusStragglers`, and this admin-driven sweep). + **StorageStats shape:** ```typescript { @@ -219,10 +222,14 @@ POST /api/admin/storage/cleanup-media { maxAgeDays: number, dryRun?: boolean } unlinkedSize: number; danglingAttachments: number; // Attachment records pointing to missing files danglingSize: number; + staleTusSessions: number; // .tus/ payload + sidecar files with mtime > 1 h old + staleTusSize: number; // Total bytes of those stale tus entries breakdown: { type: string; count: number; size: number }[]; } ``` +`staleTusSessions` / `staleTusSize` use a **fixed 1 h display threshold** (active uploads write chunks frequently; a 1 h+ gap means the user walked away). This is distinct from the `maxAgeHours` body parameter on `cleanup-tus`, which is configurable per request. + **CleanupResult shape:** ```typescript { @@ -234,7 +241,7 @@ POST /api/admin/storage/cleanup-media { maxAgeDays: number, dryRun?: boolean } } ``` -Storage functions (`getStorageStats`, `getOrphanedFiles`, `cleanupStorage`, `cleanupOldMedia`) are implemented in `utils/storageJanitor.ts`. Out of scope here -- if an uploads.md spec is created, document there. +Storage functions (`getStorageStats`, `getOrphanedFiles`, `cleanupStorage`, `cleanupOldMedia`, `cleanupStaleTusSessions`, `getStaleTusInfo`) are implemented in `utils/storageJanitor.ts`. Tus-specific lifecycle details live in `docs/systems/uploads.md` §Janitor. **Cleanup flow (UI):** 1. Admin clicks "Preview Cleanup" -- calls `cleanupStorage(dryRun=true)` or `cleanupOldMedia(days, dryRun=true)` diff --git a/docs/systems/uploads.md b/docs/systems/uploads.md index 1a8ea733..777da120 100644 --- a/docs/systems/uploads.md +++ b/docs/systems/uploads.md @@ -65,11 +65,16 @@ When the final PATCH completes, the hook: ### Janitor -| Function | Sweeps | -|----------|--------| -| `cleanupTusUploads()` | Invokes `@tus/file-store.deleteExpired()` (24 h `Upload-Expires`). | -| `cleanupTusStragglers()` | Defensive sweep of `.tus/` for files older than 48 h that the tus library failed to clean up. | -| `getUnlinkedAttachments()` | Existing 1 h grace still applies to abandoned post-finish attachments. | +| Trigger | Function / Path | Sweeps | +|---------|-----------------|--------| +| User cancels mid-upload | Client `tus.abort(true)` → tus DELETE | Immediate cleanup of the `.tus/` payload + sidecar. | +| User discards a paused/failed bubble | `transferStore.abortUpload` → manual `fetch DELETE` (when no live tus instance) | Immediate cleanup of the `.tus/` payload + sidecar. | +| Janitor tick (every ~30 s) | `cleanupTusUploads()` | Invokes `@tus/file-store.deleteExpired()` (24 h `Upload-Expires` default, configurable via `tusExpirationMs`). | +| Janitor tick (every ~30 s) | `cleanupTusStragglers()` | Defensive unlink of any `.tus/` entry whose mtime is older than `tusStragglerSweepMs` (48 h default) — catches orphans the tus library missed (payload without sidecar, sidecar without payload). | +| Admin-triggered | `POST /api/admin/storage/cleanup-tus` → `cleanupStaleTusSessions(thresholdMs, dryRun)` | Manual sweep with configurable `maxAgeHours` (default 1 h). Supports preview (`dryRun=true`) before live deletion. | +| Janitor tick (post-finalize) | `getUnlinkedAttachments()` | 1 h grace for finalized attachment rows that were never linked to a message. | + +Stats: `getStorageStats()` exposes `staleTusSessions` + `staleTusSize` for the admin Storage Overview, computed via `getStaleTusInfo(60 * 60 * 1000)` — entries with mtime older than 1 h. The display threshold is fixed (matches the cleanup default); the admin route's `maxAgeHours` is what's actually configurable. ### Security @@ -449,6 +454,7 @@ The admin routes (`routes/admin.ts`) expose the janitor functions via REST: | `GET /api/admin/storage/orphans` | GET | `getOrphanedFiles()` | | `POST /api/admin/storage/cleanup` | POST | `cleanupStorage(dryRun)` | | `POST /api/admin/storage/cleanup-media` | POST | `cleanupOldMedia(maxAgeDays, dryRun)` | +| `POST /api/admin/storage/cleanup-tus` | POST | `cleanupStaleTusSessions(maxAgeHours * 3600 * 1000, dryRun)` | All require JWT + admin role. See `docs/systems/api.md` for request/response formats. diff --git a/packages/server/src/routes/admin.test.ts b/packages/server/src/routes/admin.test.ts new file mode 100644 index 00000000..fb64a0c9 --- /dev/null +++ b/packages/server/src/routes/admin.test.ts @@ -0,0 +1,234 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import Fastify, { type FastifyInstance } from 'fastify'; +import Database from 'better-sqlite3'; +import { drizzle } from 'drizzle-orm/better-sqlite3'; +import fs from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; +import crypto from 'node:crypto'; +import { fileURLToPath } from 'node:url'; +import * as schema from '../db/schema.js'; +import { setWorkerId } from '../utils/snowflake.js'; +import { signJwt } from '../utils/auth.js'; + +setWorkerId(11); +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +type TestDb = ReturnType>; +let sqlite: Database.Database; +let testDb: TestDb; +let app: FastifyInstance; +let tusTmpDir: string; + +vi.mock('../db/index.js', () => ({ + getDb: () => testDb, + getRawDb: () => sqlite, + schema, +})); + +vi.mock('../config.js', async () => { + const real = await import('../config.js'); + return { + config: new Proxy(real.config, { + get(target, prop: string) { + if (prop === 'tusUploadDir') return tusTmpDir; + return (target as Record)[prop]; + }, + }), + }; +}); + +function applyMigrations(db: Database.Database): void { + const migrationsDir = path.resolve(__dirname, '../../drizzle'); + const files = fs.readdirSync(migrationsDir).filter(f => f.endsWith('.sql')).sort(); + for (const f of files) { + const sqlText = fs.readFileSync(path.join(migrationsDir, f), 'utf8'); + const statements = sqlText.split(/-->\s*statement-breakpoint/); + for (const stmt of statements) { + const clean = stmt.trim(); + if (clean) db.exec(clean); + } + } +} + +async function buildApp(): Promise { + const { adminRoutes } = await import('./admin.js'); + const f = Fastify(); + await f.register(adminRoutes); + return f; +} + +const ADMIN_ID = 'admin-1'; +const USER_ID = 'user-1'; +const ADMIN_USERNAME = 'admin'; +const USER_USERNAME = 'normie'; + +beforeEach(async () => { + sqlite = new Database(':memory:'); + sqlite.pragma('foreign_keys = ON'); + applyMigrations(sqlite); + testDb = drizzle(sqlite, { schema }); + + testDb.insert(schema.users).values([ + { + id: ADMIN_ID, + username: ADMIN_USERNAME, + passwordHash: 'x', + isAdmin: 1, + createdAt: Date.now(), + }, + { + id: USER_ID, + username: USER_USERNAME, + passwordHash: 'x', + isAdmin: 0, + createdAt: Date.now(), + }, + ]).run(); + + tusTmpDir = path.join(os.tmpdir(), `backspace-admin-tus-${crypto.randomBytes(8).toString('hex')}`); + app = await buildApp(); +}); + +afterEach(() => { + if (fs.existsSync(tusTmpDir)) { + fs.rmSync(tusTmpDir, { recursive: true, force: true }); + } +}); + +function adminToken(): string { + return signJwt({ userId: ADMIN_ID, username: ADMIN_USERNAME }); +} + +function userToken(): string { + return signJwt({ userId: USER_ID, username: USER_USERNAME }); +} + +describe('POST /api/admin/storage/cleanup-tus', () => { + it('rejects unauthenticated requests with 401', async () => { + const res = await app.inject({ + method: 'POST', + url: '/api/admin/storage/cleanup-tus', + payload: { maxAgeHours: 1, dryRun: true }, + }); + expect(res.statusCode).toBe(401); + }); + + it('rejects non-admin requests with 403', async () => { + const res = await app.inject({ + method: 'POST', + url: '/api/admin/storage/cleanup-tus', + headers: { Authorization: `Bearer ${userToken()}` }, + payload: { maxAgeHours: 1, dryRun: true }, + }); + expect(res.statusCode).toBe(403); + }); + + it('returns 400 when maxAgeHours is zero', async () => { + const res = await app.inject({ + method: 'POST', + url: '/api/admin/storage/cleanup-tus', + headers: { Authorization: `Bearer ${adminToken()}` }, + payload: { maxAgeHours: 0, dryRun: true }, + }); + expect(res.statusCode).toBe(400); + }); + + it('returns 400 when maxAgeHours is negative', async () => { + const res = await app.inject({ + method: 'POST', + url: '/api/admin/storage/cleanup-tus', + headers: { Authorization: `Bearer ${adminToken()}` }, + payload: { maxAgeHours: -3, dryRun: true }, + }); + expect(res.statusCode).toBe(400); + }); + + it('returns 400 when maxAgeHours is NaN/non-finite', async () => { + const res = await app.inject({ + method: 'POST', + url: '/api/admin/storage/cleanup-tus', + headers: { Authorization: `Bearer ${adminToken()}` }, + payload: { maxAgeHours: 'banana', dryRun: true }, + }); + expect(res.statusCode).toBe(400); + }); + + it('returns CleanupResult shape with zeros when .tus/ is empty', async () => { + const res = await app.inject({ + method: 'POST', + url: '/api/admin/storage/cleanup-tus', + headers: { Authorization: `Bearer ${adminToken()}` }, + payload: { maxAgeHours: 1, dryRun: false }, + }); + expect(res.statusCode).toBe(200); + const body = res.json(); + expect(body).toMatchObject({ + dryRun: false, + deletedFiles: 0, + freedBytes: 0, + deletedAttachmentRecords: 0, + errors: [], + }); + }); + + it('dryRun=true returns counts without unlinking', async () => { + fs.mkdirSync(tusTmpDir, { recursive: true }); + const stale = path.join(tusTmpDir, 'stale-session'); + fs.writeFileSync(stale, 'x'.repeat(64)); + const twoHoursAgo = Date.now() - 2 * 60 * 60 * 1000; + fs.utimesSync(stale, twoHoursAgo / 1000, twoHoursAgo / 1000); + + const res = await app.inject({ + method: 'POST', + url: '/api/admin/storage/cleanup-tus', + headers: { Authorization: `Bearer ${adminToken()}` }, + payload: { maxAgeHours: 1, dryRun: true }, + }); + expect(res.statusCode).toBe(200); + const body = res.json(); + expect(body.dryRun).toBe(true); + expect(body.deletedFiles).toBe(1); + expect(body.freedBytes).toBe(64); + expect(fs.existsSync(stale)).toBe(true); + }); + + it('dryRun=false unlinks stale entries', async () => { + fs.mkdirSync(tusTmpDir, { recursive: true }); + const stale = path.join(tusTmpDir, 'stale-session'); + fs.writeFileSync(stale, 'y'.repeat(128)); + const threeHoursAgo = Date.now() - 3 * 60 * 60 * 1000; + fs.utimesSync(stale, threeHoursAgo / 1000, threeHoursAgo / 1000); + + const res = await app.inject({ + method: 'POST', + url: '/api/admin/storage/cleanup-tus', + headers: { Authorization: `Bearer ${adminToken()}` }, + payload: { maxAgeHours: 1, dryRun: false }, + }); + expect(res.statusCode).toBe(200); + const body = res.json(); + expect(body.dryRun).toBe(false); + expect(body.deletedFiles).toBe(1); + expect(body.freedBytes).toBe(128); + expect(fs.existsSync(stale)).toBe(false); + }); + + it('defaults maxAgeHours to 1 when omitted', async () => { + fs.mkdirSync(tusTmpDir, { recursive: true }); + const stale = path.join(tusTmpDir, 'stale-90min'); + fs.writeFileSync(stale, 'z'.repeat(32)); + const ninetyMinAgo = Date.now() - 90 * 60 * 1000; + fs.utimesSync(stale, ninetyMinAgo / 1000, ninetyMinAgo / 1000); + + const res = await app.inject({ + method: 'POST', + url: '/api/admin/storage/cleanup-tus', + headers: { Authorization: `Bearer ${adminToken()}` }, + payload: { dryRun: true }, + }); + expect(res.statusCode).toBe(200); + const body = res.json(); + expect(body.deletedFiles).toBe(1); + }); +}); diff --git a/packages/server/src/routes/admin.ts b/packages/server/src/routes/admin.ts index 8421b700..8ffd3c96 100644 --- a/packages/server/src/routes/admin.ts +++ b/packages/server/src/routes/admin.ts @@ -2,7 +2,7 @@ import type { FastifyInstance } from 'fastify'; import crypto from 'crypto'; import { eq, like, or, and, ne, sql, isNull, isNotNull, gte, lte, asc, desc } from 'drizzle-orm'; import { authenticate, requireAdmin, hashPassword } from '../utils/auth.js'; -import { getStorageStats, getOrphanedFiles, cleanupStorage, cleanupOldMedia } from '../utils/storageJanitor.js'; +import { getStorageStats, getOrphanedFiles, cleanupStorage, cleanupOldMedia, cleanupStaleTusSessions } from '../utils/storageJanitor.js'; import { getDb, schema } from '../db/index.js'; import { connectionManager } from '../ws/handler.js'; import { tombstoneUser, collectDeletionBroadcastTargets } from '../utils/userDeletion.js'; @@ -74,6 +74,32 @@ export async function adminRoutes(app: FastifyInstance): Promise { } }); + // POST /api/admin/storage/cleanup-tus — admin-driven sweep of stale tus + // upload sessions. Defaults: maxAgeHours=1 (matches the staleTusSessions + // display threshold), dryRun=false. Per-file unlink errors are surfaced via + // CleanupResult.errors. No DB rows touched — `.tus/` is filesystem-only. + app.post<{ Body: { maxAgeHours?: number; dryRun?: boolean } }>('/api/admin/storage/cleanup-tus', { preHandler: [authenticate, requireAdmin] }, async (request, reply) => { + try { + const rawAge = request.body?.maxAgeHours; + const maxAgeHours = rawAge === undefined ? 1 : Number(rawAge); + if (!Number.isFinite(maxAgeHours) || maxAgeHours <= 0) { + return reply.code(400).send({ error: 'maxAgeHours must be a positive finite number', statusCode: 400 }); + } + const dryRun = request.body?.dryRun ?? false; + const thresholdMs = maxAgeHours * 60 * 60 * 1000; + const result = cleanupStaleTusSessions(thresholdMs, dryRun); + return reply.code(200).send({ + dryRun, + deletedFiles: result.deletedFiles, + freedBytes: result.freedBytes, + deletedAttachmentRecords: 0, + errors: result.errors, + }); + } catch (err: any) { + return reply.code(500).send({ error: `Tus cleanup failed: ${err.message}`, statusCode: 500 }); + } + }); + // ─── User Management ──────────────────────────────────────────────────── // GET /api/admin/users — paginated user list with search diff --git a/packages/server/src/utils/storageJanitor.ts b/packages/server/src/utils/storageJanitor.ts index 5d258fac..4b9a3f1c 100644 --- a/packages/server/src/utils/storageJanitor.ts +++ b/packages/server/src/utils/storageJanitor.ts @@ -174,11 +174,22 @@ function getDanglingAttachments(): { id: string; filename: string; size: number return [...danglingSpace, ...danglingDm]; } +/** + * Threshold for the `staleTusSessions` count exposed via `getStorageStats()`. + * Distinct from `tusStragglerSweepMs` (defensive sweep, default 48h) and from + * the configurable `maxAgeHours` of the admin cleanup route — this is purely + * the "how many entries look abandoned right now?" display threshold. An + * active upload writes chunks often, so a 1h+ gap is a strong signal the user + * walked away (paused/crashed/discarded without DELETE). + */ +const STALE_TUS_DISPLAY_THRESHOLD_MS = 60 * 60 * 1000; + export function getStorageStats(): StorageStats { const diskFiles = getDiskFiles(); const referenced = getReferencedFilenames(); const unlinked = getUnlinkedAttachments(); const dangling = getDanglingAttachments(); + const staleTus = getStaleTusInfo(STALE_TUS_DISPLAY_THRESHOLD_MS); const danglingFilenames = new Set(); let danglingSize = 0; @@ -235,6 +246,8 @@ export function getStorageStats(): StorageStats { unlinkedSize, danglingAttachments: dangling.length, danglingSize, + staleTusSessions: staleTus.count, + staleTusSize: staleTus.size, breakdown, }; } @@ -721,19 +734,33 @@ export async function cleanupTusUploads(): Promise<{ removed: number }> { } /** - * Defensive sweep of `${config.tusUploadDir}`: any file (payload OR sidecar) - * whose mtime is older than `config.tusStragglerSweepMs` is unlinked. Covers - * the rare case where a tus crash left an orphan that deleteExpired() doesn't - * recognize — e.g. a payload without sidecar (so creation_date is unknown), - * or a sidecar without payload (so getUpload() rejects). + * Walk `config.tusUploadDir`, yielding `{ name, full, size, mtimeMs }` for each + * regular file matching `predicate`. Tolerates missing dir (yields nothing) and + * skips entries whose `statSync` throws (e.g. file vanished mid-walk). + * + * Shared between the unconditional straggler sweep, the admin-driven + * stale-session cleanup, and the stats helper. Centralising the iteration + * keeps the .tus/ semantics (which entries count as "files we care about") in + * exactly one place. */ -export function cleanupTusStragglers(): { removed: number } { - if (!fs.existsSync(config.tusUploadDir)) return { removed: 0 }; - const entries = fs.readdirSync(config.tusUploadDir); - const cutoff = Date.now() - config.tusStragglerSweepMs; - let removed = 0; - for (const entry of entries) { - const full = path.join(config.tusUploadDir, entry); +interface TusEntry { + name: string; + full: string; + size: number; + mtimeMs: number; +} + +function walkTusDir(predicate: (entry: TusEntry) => boolean): TusEntry[] { + if (!fs.existsSync(config.tusUploadDir)) return []; + let names: string[]; + try { + names = fs.readdirSync(config.tusUploadDir); + } catch { + return []; + } + const out: TusEntry[] = []; + for (const name of names) { + const full = path.join(config.tusUploadDir, name); let stat: fs.Stats; try { stat = fs.statSync(full); @@ -741,15 +768,90 @@ export function cleanupTusStragglers(): { removed: number } { continue; } if (!stat.isFile()) continue; - if (stat.mtimeMs >= cutoff) continue; - try { - fs.unlinkSync(full); - removed += 1; - } catch { - // Race with tus's own cleanup or permissions error — ignore - } + const entry: TusEntry = { name, full, size: stat.size, mtimeMs: stat.mtimeMs }; + if (predicate(entry)) out.push(entry); } - return { removed }; + return out; +} + +/** + * Inspect `.tus/` for entries whose mtime is older than `thresholdMs`. Returns + * an aggregate snapshot — count, total bytes, oldest mtime — without touching + * any files. Used by `getStorageStats()` to surface a count of "abandoned" + * tus sessions in the admin UI, and by the admin route as a dry-run primitive. + * + * "Stale" here is purely mtime-based and counts both payloads and `.json` + * sidecars; the conservative threshold (1h) used by `getStorageStats` matches + * the intuition that an active upload writes chunks frequently, so a 1h+ gap + * means the user genuinely walked away. + */ +export function getStaleTusInfo(thresholdMs: number): { count: number; size: number; oldestAt: number | null } { + const cutoff = Date.now() - thresholdMs; + let count = 0; + let size = 0; + let oldestAt: number | null = null; + walkTusDir((entry) => { + if (entry.mtimeMs >= cutoff) return false; + count += 1; + size += entry.size; + if (oldestAt === null || entry.mtimeMs < oldestAt) oldestAt = entry.mtimeMs; + return false; // we don't need the returned array, just side-effects + }); + return { count, size, oldestAt }; +} + +/** + * Admin-driven sweep of stale tus sessions. Walks `.tus/`, finds entries with + * mtime older than `thresholdMs`, optionally unlinks each one, and returns an + * aggregate result. In `dryRun=true` mode no files are touched. Per-file + * unlink errors are collected (not thrown) so a single bad entry can't abort + * the whole sweep. + * + * Note that `.tus/` holds *pairs* (payload + `.json` sidecar) per session, but + * we treat each file independently — the sidecar's mtime is updated by + * `@tus/file-store` on every PATCH, so payload + sidecar move together; if the + * pair is genuinely abandoned, both are stale and both get reaped. No need for + * pair reconciliation. + */ +export function cleanupStaleTusSessions( + thresholdMs: number, + dryRun: boolean, +): { deletedFiles: number; freedBytes: number; errors: string[] } { + const cutoff = Date.now() - thresholdMs; + const stale = walkTusDir((entry) => entry.mtimeMs < cutoff); + const errors: string[] = []; + let deletedFiles = 0; + let freedBytes = 0; + for (const entry of stale) { + if (!dryRun) { + try { + fs.unlinkSync(entry.full); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + errors.push(`Failed to delete ${entry.name}: ${message}`); + continue; + } + } + deletedFiles += 1; + freedBytes += entry.size; + } + return { deletedFiles, freedBytes, errors }; +} + +/** + * Defensive sweep of `${config.tusUploadDir}`: any file (payload OR sidecar) + * whose mtime is older than `thresholdMs` (default `config.tusStragglerSweepMs`, + * 48 h) is unlinked. Covers the rare case where a tus crash left an orphan + * that deleteExpired() doesn't recognize — e.g. a payload without sidecar (so + * creation_date is unknown), or a sidecar without payload (so getUpload() + * rejects). + * + * Janitor-tick contract preserved: returns `{ removed }` with the count of + * files actually unlinked. Internally delegates to `cleanupStaleTusSessions`. + */ +export function cleanupTusStragglers(thresholdMs: number = config.tusStragglerSweepMs): { removed: number } { + const result = cleanupStaleTusSessions(thresholdMs, false); + return { removed: result.deletedFiles }; } // cleanupStorage is expensive (full disk scan + DB joins) so we run it at diff --git a/packages/server/src/utils/storageJanitor.tus.test.ts b/packages/server/src/utils/storageJanitor.tus.test.ts index 58abf022..995b2e85 100644 --- a/packages/server/src/utils/storageJanitor.tus.test.ts +++ b/packages/server/src/utils/storageJanitor.tus.test.ts @@ -78,4 +78,133 @@ describe('cleanupTusStragglers', () => { expect(result.removed).toBe(0); }); + + it('honours an explicit threshold override', async () => { + const { cleanupTusStragglers } = await import('./storageJanitor.js'); + fs.mkdirSync(tmpDir, { recursive: true }); + const file = path.join(tmpDir, 'two-hour-old'); + fs.writeFileSync(file, 'data'); + const twoHoursAgo = Date.now() - 2 * 60 * 60 * 1000; + const seconds = twoHoursAgo / 1000; + fs.utimesSync(file, seconds, seconds); + + // Default 48h threshold would skip this file. Override to 1h to catch it. + const result = cleanupTusStragglers(60 * 60 * 1000); + + expect(result.removed).toBe(1); + expect(fs.existsSync(file)).toBe(false); + }); +}); + +describe('getStaleTusInfo', () => { + it('returns zeros when the directory does not exist', async () => { + const { getStaleTusInfo } = await import('./storageJanitor.js'); + expect(fs.existsSync(tmpDir)).toBe(false); + + const info = getStaleTusInfo(60 * 60 * 1000); + + expect(info).toEqual({ count: 0, size: 0, oldestAt: null }); + }); + + it('excludes entries newer than the threshold', async () => { + const { getStaleTusInfo } = await import('./storageJanitor.js'); + fs.mkdirSync(tmpDir, { recursive: true }); + fs.writeFileSync(path.join(tmpDir, 'fresh'), 'recent'); + // mtime defaults to "now" — within any reasonable threshold. + + const info = getStaleTusInfo(60 * 60 * 1000); + + expect(info.count).toBe(0); + expect(info.size).toBe(0); + expect(info.oldestAt).toBeNull(); + }); + + it('includes entries older than the threshold and tracks oldest mtime', async () => { + const { getStaleTusInfo } = await import('./storageJanitor.js'); + fs.mkdirSync(tmpDir, { recursive: true }); + + const stale1 = path.join(tmpDir, 'stale-1'); + const stale2 = path.join(tmpDir, 'stale-2'); + const fresh = path.join(tmpDir, 'fresh'); + fs.writeFileSync(stale1, 'a'.repeat(100)); + fs.writeFileSync(stale2, 'b'.repeat(250)); + fs.writeFileSync(fresh, 'c'.repeat(50)); + + const twoHoursAgo = Date.now() - 2 * 60 * 60 * 1000; + const fourHoursAgo = Date.now() - 4 * 60 * 60 * 1000; + fs.utimesSync(stale1, twoHoursAgo / 1000, twoHoursAgo / 1000); + fs.utimesSync(stale2, fourHoursAgo / 1000, fourHoursAgo / 1000); + + const info = getStaleTusInfo(60 * 60 * 1000); // 1h threshold + + expect(info.count).toBe(2); + expect(info.size).toBe(350); + expect(info.oldestAt).not.toBeNull(); + // Oldest mtime should be ~ fourHoursAgo (within fs precision, allow 1.5s slack) + expect(Math.abs((info.oldestAt ?? 0) - fourHoursAgo)).toBeLessThan(1500); + }); + + it('skips subdirectories', async () => { + const { getStaleTusInfo } = await import('./storageJanitor.js'); + fs.mkdirSync(tmpDir, { recursive: true }); + const sub = path.join(tmpDir, 'subdir'); + fs.mkdirSync(sub); + const oneDayAgo = Date.now() - 24 * 60 * 60 * 1000; + fs.utimesSync(sub, oneDayAgo / 1000, oneDayAgo / 1000); + + const info = getStaleTusInfo(60 * 60 * 1000); + + expect(info.count).toBe(0); + }); +}); + +describe('cleanupStaleTusSessions', () => { + it('returns zero counts when the directory does not exist', async () => { + const { cleanupStaleTusSessions } = await import('./storageJanitor.js'); + expect(fs.existsSync(tmpDir)).toBe(false); + + const result = cleanupStaleTusSessions(60 * 60 * 1000, false); + + expect(result.deletedFiles).toBe(0); + expect(result.freedBytes).toBe(0); + expect(result.errors).toEqual([]); + }); + + it('counts but does not delete when dryRun=true', async () => { + const { cleanupStaleTusSessions } = await import('./storageJanitor.js'); + fs.mkdirSync(tmpDir, { recursive: true }); + const stale = path.join(tmpDir, 'stale'); + fs.writeFileSync(stale, 'x'.repeat(200)); + const twoHoursAgo = Date.now() - 2 * 60 * 60 * 1000; + fs.utimesSync(stale, twoHoursAgo / 1000, twoHoursAgo / 1000); + + const result = cleanupStaleTusSessions(60 * 60 * 1000, true); + + expect(result.deletedFiles).toBe(1); + expect(result.freedBytes).toBe(200); + expect(result.errors).toEqual([]); + // File must still exist on disk. + expect(fs.existsSync(stale)).toBe(true); + }); + + it('unlinks stale entries when dryRun=false and leaves fresh ones alone', async () => { + const { cleanupStaleTusSessions } = await import('./storageJanitor.js'); + fs.mkdirSync(tmpDir, { recursive: true }); + + const stale = path.join(tmpDir, 'stale'); + const fresh = path.join(tmpDir, 'fresh'); + fs.writeFileSync(stale, 'x'.repeat(123)); + fs.writeFileSync(fresh, 'y'.repeat(456)); + + const threeHoursAgo = Date.now() - 3 * 60 * 60 * 1000; + fs.utimesSync(stale, threeHoursAgo / 1000, threeHoursAgo / 1000); + + const result = cleanupStaleTusSessions(60 * 60 * 1000, false); + + expect(result.deletedFiles).toBe(1); + expect(result.freedBytes).toBe(123); + expect(result.errors).toEqual([]); + expect(fs.existsSync(stale)).toBe(false); + expect(fs.existsSync(fresh)).toBe(true); + }); }); diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 060d7e22..43096358 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -820,6 +820,10 @@ export interface StorageStats { unlinkedSize: number; danglingAttachments: number; danglingSize: number; + /** Count of `.tus/` entries (payloads + sidecars) with mtime older than 1h. */ + staleTusSessions: number; + /** Total size in bytes of those stale `.tus/` entries. */ + staleTusSize: number; breakdown: StorageBreakdown[]; } diff --git a/packages/web/src/api/client.ts b/packages/web/src/api/client.ts index 804c6178..27436060 100644 --- a/packages/web/src/api/client.ts +++ b/packages/web/src/api/client.ts @@ -264,6 +264,7 @@ export class BackspaceApiClient { storageOrphans: () => Promise<{ orphans: OrphanedFile[] }>; storageCleanup: (dryRun?: boolean) => Promise; cleanupOldMedia: (maxAgeDays: number, dryRun?: boolean) => Promise; + cleanupTusSessions: (maxAgeHours: number, dryRun?: boolean) => Promise; listUsers: (params?: { q?: string; page?: number; pageSize?: number; showDeleted?: boolean; homeInstance?: string; role?: string; joinedAfter?: string; joinedBefore?: string; sort?: string }) => Promise; listInstances: () => Promise<{ instances: string[] }>; setUserRole: (userId: string, isAdmin: boolean) => Promise; @@ -687,6 +688,8 @@ export class BackspaceApiClient { storageCleanup: (dryRun = false) => request('POST', '/admin/storage/cleanup', { dryRun }), cleanupOldMedia: (maxAgeDays: number, dryRun = false) => request('POST', '/admin/storage/cleanup-media', { maxAgeDays, dryRun }), + cleanupTusSessions: (maxAgeHours: number, dryRun = false) => + request('POST', '/admin/storage/cleanup-tus', { maxAgeHours, dryRun }), listUsers: (params) => { const qs = new URLSearchParams(); if (params?.q) qs.set('q', params.q); diff --git a/packages/web/src/components/modals/instanceSettingsPanels/StoragePanel.tsx b/packages/web/src/components/modals/instanceSettingsPanels/StoragePanel.tsx index c0d758d5..a9213608 100644 --- a/packages/web/src/components/modals/instanceSettingsPanels/StoragePanel.tsx +++ b/packages/web/src/components/modals/instanceSettingsPanels/StoragePanel.tsx @@ -50,6 +50,12 @@ export function StoragePanel() { const [mediaCleaning, setMediaCleaning] = useState(false); const [mediaPreviewDone, setMediaPreviewDone] = useState(false); + // Stale tus session cleanup state + const [tusMaxAgeHours, setTusMaxAgeHours] = useState(1); + const [tusCleanupResult, setTusCleanupResult] = useState(null); + const [tusCleaning, setTusCleaning] = useState(false); + const [tusPreviewDone, setTusPreviewDone] = useState(false); + const fetchStats = useCallback(async () => { setLoading(true); setLoadError(''); @@ -123,6 +129,27 @@ export function StoragePanel() { } }; + const handleTusCleanup = async (dryRun: boolean) => { + if (!Number.isFinite(tusMaxAgeHours) || tusMaxAgeHours <= 0) return; + setTusCleaning(true); + setTusCleanupResult(null); + try { + const result = await api.admin.cleanupTusSessions(tusMaxAgeHours, dryRun); + setTusCleanupResult(result); + if (dryRun) { + setTusPreviewDone(true); + } else { + setTusPreviewDone(false); + addToast(`Cleaned ${result.deletedFiles} stale upload session${result.deletedFiles !== 1 ? 's' : ''} (${formatBytes(result.freedBytes)})`, 'success'); + await fetchStats(); + } + } catch (err) { + addToast(err instanceof Error ? err.message : 'Stale upload cleanup failed', 'warning'); + } finally { + setTusCleaning(false); + } + }; + const handleCleanup = async (dryRun: boolean) => { setCleaning(true); setCleanupResult(null); @@ -202,6 +229,13 @@ export function StoragePanel() {
{formatBytes(stats.danglingSize)}
+
+
Stale Uploads
+
0 ? 'text-accent-amber' : 'text-txt-primary'}`}> + {stats.staleTusSessions} +
+
{formatBytes(stats.staleTusSize)}
+
@@ -317,6 +351,69 @@ export function StoragePanel() { + {/* Stale Uploads */} +
+
Stale Uploads
+
+
+ Abandoned tus upload sessions in .tus/ (paused/crashed without DELETE). Auto-expire runs every 24 hours; this lets you sweep proactively. +
+
+ + { + const next = Number(e.target.value); + setTusMaxAgeHours(Number.isFinite(next) && next > 0 ? next : 0); + setTusPreviewDone(false); + setTusCleanupResult(null); + }} + className="input-standard w-24 px-2 py-1 text-sm text-center" + /> +
+ +
+ + +
+ + {tusCleanupResult && ( +
+
+ {tusCleanupResult.dryRun ? 'Preview — no files deleted' : 'Cleanup complete'} +
+
+ {tusCleanupResult.deletedFiles} session file{tusCleanupResult.deletedFiles !== 1 ? 's' : ''} ({formatBytes(tusCleanupResult.freedBytes)}) +
+ {tusCleanupResult.errors.length > 0 && ( +
+ {tusCleanupResult.errors.length} error{tusCleanupResult.errors.length !== 1 ? 's' : ''}: {tusCleanupResult.errors[0]} +
+ )} +
+ )} +
+
+ {/* Media Retention */}
Media Retention