fix(uploads): graceful fallback for browser-unplayable video (HEVC .mov)

macOS screen recordings are HEVC inside a .mov container, which Chromium,
Firefox and stock Electron can't decode. The file uploaded fine and a
server-side ffmpeg poster was generated, but inline <video> playback failed
silently — stuck at 0:00 with no error, since AttachmentRenderer had no error
handling. Root cause: the system had no concept of web-playability.

Server detects, client degrades:
- mediaPlayable.ts: classifyVideoPlayable(mimetype, codec) — tri-state
  (false = known-undecodable e.g. HEVC/ProRes, true = web codec in web
  container, null = unknown/optimistic). Never widens `false` beyond codecs
  that fail everywhere, so ffmpeg-less instances keep prior behaviour.
- probeMediaMeta now captures the video codec_name; the upload finish hook
  stores the verdict in the new attachments.playable column (migration 0007).
- Flag propagated through every serializer: space messages, DMs, WS, and
  federation relay (outbound + inbound) — federation-compatible.
- VideoAttachment component: playable===false renders a download card (poster
  + "Can't play here — download" + name/duration/size) with no dead-player
  flash; otherwise plays inline with an onError fallback to the same card.

Specs updated: uploads.md, database.md, federation.md.
This commit is contained in:
Jannis Braun
2026-06-30 17:38:11 +02:00
parent e84daf57aa
commit 209aef7e9d
18 changed files with 3999 additions and 30 deletions
+4
View File
@@ -102,6 +102,10 @@ export const attachments = sqliteTable('attachments', {
width: integer('width'),
height: integer('height'),
duration: real('duration'),
// Tri-state web-playability for video attachments: 1 = decodable in a
// browser <video>, 0 = known-undecodable (e.g. HEVC .mov), NULL = unknown
// (codec unprobed / non-video). Drives the client's download fallback.
playable: integer('playable', { mode: 'boolean' }),
sourceUrl: text('source_url'),
federationStatus: text('federation_status'),
federationMeta: text('federation_meta'),
+1
View File
@@ -122,6 +122,7 @@ export function buildDmMessageWithUser(
width: a.width ?? null,
height: a.height ?? null,
duration: a.duration ?? null,
playable: a.playable ?? null,
federationStatus: a.federationStatus ?? null,
federationMeta: a.federationMeta ?? null,
createdAt: a.createdAt,
+3
View File
@@ -2911,6 +2911,7 @@ export async function federationRoutes(app: FastifyInstance): Promise<void> {
width: a.width ?? undefined,
height: a.height ?? undefined,
duration: a.duration ?? undefined,
playable: a.playable ?? null,
thumbnailFilename: a.thumbnailFilename ?? undefined,
sourceUrl: `${localOrigin}/api/uploads/${a.filename}`,
}));
@@ -3656,6 +3657,7 @@ async function processCreateEvent(
width: attachment.width ?? null,
height: attachment.height ?? null,
duration: attachment.duration ?? null,
playable: attachment.playable ?? null,
thumbnailFilename: null, // Don't copy source thumbnail — it doesn't exist locally
sourceUrl: attachment.sourceUrl,
createdAt: now,
@@ -3830,6 +3832,7 @@ function processUpdateEvent(
width: a.width,
height: a.height,
duration: a.duration,
playable: a.playable ?? null,
createdAt: a.createdAt,
}));
+9 -1
View File
@@ -20,6 +20,7 @@ import {
generateVideoThumbnail,
thumbFilename,
} from '../utils/thumbnail.js';
import { classifyVideoPlayable } from '../utils/mediaPlayable.js';
import { eq } from 'drizzle-orm';
import type { Attachment } from '@backspace/shared';
import fs from 'node:fs';
@@ -251,6 +252,10 @@ export async function filesRoutes(app: FastifyInstance): Promise<void> {
let width: number | null = null;
let height: number | null = null;
let duration: number | null = null;
// Tri-state web-playability for video (null = unknown/optimistic). Lets
// the client render a download fallback for codecs the browser can't
// decode (e.g. HEVC .mov) instead of a dead <video> stuck at 0:00.
let playable: boolean | null = null;
try {
if (isResizableImage(mimetype)) {
stagedThumbName = await generateThumbnail(srcPath, mimetype, config.uploadDir);
@@ -271,6 +276,7 @@ export async function filesRoutes(app: FastifyInstance): Promise<void> {
duration = mediaMeta.duration ?? null;
if (width === null && mediaMeta.width) width = mediaMeta.width;
if (height === null && mediaMeta.height) height = mediaMeta.height;
playable = classifyVideoPlayable(mimetype, mediaMeta.codec);
}
} else if (mimetype.startsWith('audio/')) {
const mediaMeta = await probeMediaMeta(srcPath, mimetype);
@@ -286,7 +292,7 @@ export async function filesRoutes(app: FastifyInstance): Promise<void> {
} catch { /* ignore */ }
stagedThumbName = null;
}
width = null; height = null; duration = null;
width = null; height = null; duration = null; playable = null;
}
// ── Commit point: rename .tus/<id> → uploads/<snowflakeId><ext> ──────
@@ -337,6 +343,7 @@ export async function filesRoutes(app: FastifyInstance): Promise<void> {
width,
height,
duration,
playable,
createdAt: now,
}).run();
} catch (err) {
@@ -362,6 +369,7 @@ export async function filesRoutes(app: FastifyInstance): Promise<void> {
width: width ?? undefined,
height: height ?? undefined,
duration: duration ?? undefined,
playable,
createdAt: now,
};
+2
View File
@@ -118,6 +118,7 @@ export function fetchReplyToMessages(messages: (typeof schema.messages.$inferSel
width: a.width ?? null,
height: a.height ?? null,
duration: a.duration ?? null,
playable: a.playable ?? null,
createdAt: a.createdAt,
})),
embeds: [],
@@ -156,6 +157,7 @@ export function buildMessageWithUser(
width: a.width ?? null,
height: a.height ?? null,
duration: a.duration ?? null,
playable: a.playable ?? null,
federationStatus: a.federationStatus ?? null,
federationMeta: a.federationMeta ?? null,
createdAt: a.createdAt,
@@ -446,6 +446,7 @@ export function queueDmRelay(
width: a.width ?? undefined,
height: a.height ?? undefined,
duration: a.duration ?? undefined,
playable: a.playable ?? null,
thumbnailFilename: a.thumbnailFilename ?? undefined,
sourceUrl: `${domainOrigin}/api/uploads/${a.filename}`,
}));
@@ -0,0 +1,50 @@
import { describe, it, expect } from 'vitest';
import { classifyVideoPlayable } from './mediaPlayable';
describe('classifyVideoPlayable', () => {
it('marks HEVC as not web-playable regardless of container', () => {
// macOS screen recordings: video/quicktime + hevc — the reported bug.
expect(classifyVideoPlayable('video/quicktime', 'hevc')).toBe(false);
// HEVC inside an mp4 container is equally undecodable in Chromium/Firefox.
expect(classifyVideoPlayable('video/mp4', 'hevc')).toBe(false);
// ffprobe sometimes reports the tag rather than the codec name.
expect(classifyVideoPlayable('video/quicktime', 'hvc1')).toBe(false);
expect(classifyVideoPlayable('video/quicktime', 'hev1')).toBe(false);
});
it('marks other known-undecodable codecs as not web-playable', () => {
expect(classifyVideoPlayable('video/quicktime', 'prores')).toBe(false);
expect(classifyVideoPlayable('video/x-msvideo', 'wmv3')).toBe(false);
});
it('marks web-standard codec in a web container as playable', () => {
expect(classifyVideoPlayable('video/mp4', 'h264')).toBe(true);
expect(classifyVideoPlayable('video/webm', 'vp9')).toBe(true);
expect(classifyVideoPlayable('video/webm', 'av1')).toBe(true);
// ffprobe reports H.264 as 'h264'; the mp4 box tag is 'avc1'.
expect(classifyVideoPlayable('video/mp4', 'avc1')).toBe(true);
});
it('is optimistic (null) for web-safe codec in a non-web container', () => {
// H.264 in a .mov plays in Chrome/Safari but not Firefox — let the client
// attempt playback and fall back via onError rather than blocking it.
expect(classifyVideoPlayable('video/quicktime', 'h264')).toBeNull();
});
it('is optimistic (null) when the codec is unknown', () => {
// ffmpeg unavailable / probe failed — must not regress to "unplayable".
expect(classifyVideoPlayable('video/mp4', undefined)).toBeNull();
expect(classifyVideoPlayable('video/mp4', null)).toBeNull();
expect(classifyVideoPlayable('video/mp4', '')).toBeNull();
});
it('returns null for non-video mimetypes', () => {
expect(classifyVideoPlayable('audio/mpeg', 'mp3')).toBeNull();
expect(classifyVideoPlayable('image/png', undefined)).toBeNull();
});
it('is case-insensitive on the codec name', () => {
expect(classifyVideoPlayable('video/quicktime', 'HEVC')).toBe(false);
expect(classifyVideoPlayable('video/mp4', 'H264')).toBe(true);
});
});
@@ -0,0 +1,77 @@
/**
* Web-playability classification for video attachments.
*
* The browser `<video>` element can only decode a subset of the formats users
* upload. The dominant failure case is a macOS screen recording — a
* `video/quicktime` (.mov) container holding an HEVC (H.265) stream — which
* Chromium, Firefox and stock Electron cannot decode. The file uploads fine,
* a server-side ffmpeg poster is generated, but inline playback silently fails
* (stuck at 0:00 with no error). We persist this classification per attachment
* so the client can render a download fallback instead of a dead player.
*
* The result is a deliberate tri-state:
* - `false` — confidently undecodable in mainstream browsers (e.g. HEVC).
* The client renders the fallback card directly, no flash.
* - `true` — confidently decodable (web codec in a web container).
* - `null` — unknown / optimistic. The codec couldn't be probed (ffmpeg
* absent or probe failed), or it's a web-safe codec in a
* container with inconsistent cross-browser support (e.g. H.264
* in .mov). The client attempts playback and degrades to the
* fallback via the `<video>` `onError` handler.
*
* We never widen `false` beyond codecs we are certain fail everywhere, so an
* instance without ffmpeg (codec always undefined) keeps today's behaviour
* (attempt playback) rather than regressing every video to "unplayable".
*/
/** ffprobe `codec_name` (or mp4/mov box tag) values that no mainstream browser decodes. */
const UNPLAYABLE_VIDEO_CODECS = new Set([
// HEVC / H.265 — codec_name is `hevc`; box tags are `hvc1` / `hev1`.
'hevc', 'hvc1', 'hev1', 'h265',
// Apple ProRes — editing/intermediate codec, never web-decodable.
'prores',
// Windows Media Video.
'wmv1', 'wmv2', 'wmv3', 'vc1',
// Legacy / capture codecs.
'mpeg1video', 'mpeg2video', 'dnxhd', 'mjpeg', 'vp6', 'vp6f',
]);
/** Codecs every modern browser can decode when in a web-standard container. */
const PLAYABLE_VIDEO_CODECS = new Set([
'h264', 'avc1', // H.264 / AVC
'vp8', 'vp9',
'av1', 'av01',
'theora',
]);
/** Containers with reliable cross-browser `<video>` support. */
const PLAYABLE_VIDEO_CONTAINERS = new Set([
'video/mp4',
'video/webm',
'video/ogg',
]);
/**
* Classify whether a video attachment can be played inline in a browser
* `<video>` element, given its container mimetype and the probed video codec.
*
* @param mimetype Container mimetype (e.g. `video/quicktime`).
* @param codec ffprobe `codec_name` of the primary video stream, if known.
* @returns `false` (known-unplayable), `true` (known-playable) or `null` (unknown/optimistic).
*/
export function classifyVideoPlayable(
mimetype: string,
codec: string | null | undefined,
): boolean | null {
if (!mimetype.startsWith('video/')) return null;
if (!codec) return null;
const c = codec.toLowerCase();
if (UNPLAYABLE_VIDEO_CODECS.has(c)) return false;
if (PLAYABLE_VIDEO_CONTAINERS.has(mimetype) && PLAYABLE_VIDEO_CODECS.has(c)) return true;
// Web-safe codec in a shaky container (H.264 in .mov), or an unrecognised
// codec we can't vouch for: stay optimistic and let the client's onError
// handler catch a genuine playback failure.
return null;
}
+10 -5
View File
@@ -138,19 +138,21 @@ export async function probeImageDimensions(filepath: string): Promise<{ width: n
export async function probeMediaMeta(
filepath: string,
mimetype: string,
): Promise<{ width?: number; height?: number; duration?: number } | null> {
): Promise<{ width?: number; height?: number; duration?: number; codec?: string } | null> {
if (!(await checkFfmpeg())) return null;
try {
const result: { width?: number; height?: number; duration?: number } = {};
const result: { width?: number; height?: number; duration?: number; codec?: string } = {};
// Video dimensions (stream-level)
// Video dimensions + codec (stream-level). The codec drives web-playability
// classification (see utils/mediaPlayable.ts) so the client can render a
// download fallback for formats the browser can't decode (e.g. HEVC .mov).
if (mimetype.startsWith('video/')) {
try {
const { stdout } = await execFile('ffprobe', [
'-v', 'error',
'-select_streams', 'v:0',
'-show_entries', 'stream=width,height',
'-show_entries', 'stream=width,height,codec_name',
'-of', 'json',
'-i', filepath,
], { timeout: FFMPEG_TIMEOUT });
@@ -161,7 +163,10 @@ export async function probeMediaMeta(
result.width = stream.width;
result.height = stream.height;
}
} catch { /* dimension probe failed — continue for duration */ }
if (typeof stream?.codec_name === 'string' && stream.codec_name) {
result.codec = stream.codec_name;
}
} catch { /* dimension/codec probe failed — continue for duration */ }
}
// Duration (format-level — works for video and audio)
+1
View File
@@ -74,6 +74,7 @@ function getMessageWithUser(messageId: string): MessageWithUser | null {
width: a.width ?? null,
height: a.height ?? null,
duration: a.duration ?? null,
playable: a.playable ?? null,
createdAt: a.createdAt,
}));