fix(uploads): graceful fallback for browser-unplayable video (HEVC .mov)

macOS screen recordings are HEVC inside a .mov container, which Chromium,
Firefox and stock Electron can't decode. The file uploaded fine and a
server-side ffmpeg poster was generated, but inline <video> playback failed
silently — stuck at 0:00 with no error, since AttachmentRenderer had no error
handling. Root cause: the system had no concept of web-playability.

Server detects, client degrades:
- mediaPlayable.ts: classifyVideoPlayable(mimetype, codec) — tri-state
  (false = known-undecodable e.g. HEVC/ProRes, true = web codec in web
  container, null = unknown/optimistic). Never widens `false` beyond codecs
  that fail everywhere, so ffmpeg-less instances keep prior behaviour.
- probeMediaMeta now captures the video codec_name; the upload finish hook
  stores the verdict in the new attachments.playable column (migration 0007).
- Flag propagated through every serializer: space messages, DMs, WS, and
  federation relay (outbound + inbound) — federation-compatible.
- VideoAttachment component: playable===false renders a download card (poster
  + "Can't play here — download" + name/duration/size) with no dead-player
  flash; otherwise plays inline with an onError fallback to the same card.

Specs updated: uploads.md, database.md, federation.md.
This commit is contained in:
Jannis Braun
2026-06-30 17:38:11 +02:00
parent e84daf57aa
commit 209aef7e9d
18 changed files with 3999 additions and 30 deletions
@@ -446,6 +446,7 @@ export function queueDmRelay(
width: a.width ?? undefined,
height: a.height ?? undefined,
duration: a.duration ?? undefined,
playable: a.playable ?? null,
thumbnailFilename: a.thumbnailFilename ?? undefined,
sourceUrl: `${domainOrigin}/api/uploads/${a.filename}`,
}));
@@ -0,0 +1,50 @@
import { describe, it, expect } from 'vitest';
import { classifyVideoPlayable } from './mediaPlayable';
describe('classifyVideoPlayable', () => {
it('marks HEVC as not web-playable regardless of container', () => {
// macOS screen recordings: video/quicktime + hevc — the reported bug.
expect(classifyVideoPlayable('video/quicktime', 'hevc')).toBe(false);
// HEVC inside an mp4 container is equally undecodable in Chromium/Firefox.
expect(classifyVideoPlayable('video/mp4', 'hevc')).toBe(false);
// ffprobe sometimes reports the tag rather than the codec name.
expect(classifyVideoPlayable('video/quicktime', 'hvc1')).toBe(false);
expect(classifyVideoPlayable('video/quicktime', 'hev1')).toBe(false);
});
it('marks other known-undecodable codecs as not web-playable', () => {
expect(classifyVideoPlayable('video/quicktime', 'prores')).toBe(false);
expect(classifyVideoPlayable('video/x-msvideo', 'wmv3')).toBe(false);
});
it('marks web-standard codec in a web container as playable', () => {
expect(classifyVideoPlayable('video/mp4', 'h264')).toBe(true);
expect(classifyVideoPlayable('video/webm', 'vp9')).toBe(true);
expect(classifyVideoPlayable('video/webm', 'av1')).toBe(true);
// ffprobe reports H.264 as 'h264'; the mp4 box tag is 'avc1'.
expect(classifyVideoPlayable('video/mp4', 'avc1')).toBe(true);
});
it('is optimistic (null) for web-safe codec in a non-web container', () => {
// H.264 in a .mov plays in Chrome/Safari but not Firefox — let the client
// attempt playback and fall back via onError rather than blocking it.
expect(classifyVideoPlayable('video/quicktime', 'h264')).toBeNull();
});
it('is optimistic (null) when the codec is unknown', () => {
// ffmpeg unavailable / probe failed — must not regress to "unplayable".
expect(classifyVideoPlayable('video/mp4', undefined)).toBeNull();
expect(classifyVideoPlayable('video/mp4', null)).toBeNull();
expect(classifyVideoPlayable('video/mp4', '')).toBeNull();
});
it('returns null for non-video mimetypes', () => {
expect(classifyVideoPlayable('audio/mpeg', 'mp3')).toBeNull();
expect(classifyVideoPlayable('image/png', undefined)).toBeNull();
});
it('is case-insensitive on the codec name', () => {
expect(classifyVideoPlayable('video/quicktime', 'HEVC')).toBe(false);
expect(classifyVideoPlayable('video/mp4', 'H264')).toBe(true);
});
});
@@ -0,0 +1,77 @@
/**
* Web-playability classification for video attachments.
*
* The browser `<video>` element can only decode a subset of the formats users
* upload. The dominant failure case is a macOS screen recording — a
* `video/quicktime` (.mov) container holding an HEVC (H.265) stream — which
* Chromium, Firefox and stock Electron cannot decode. The file uploads fine,
* a server-side ffmpeg poster is generated, but inline playback silently fails
* (stuck at 0:00 with no error). We persist this classification per attachment
* so the client can render a download fallback instead of a dead player.
*
* The result is a deliberate tri-state:
* - `false` — confidently undecodable in mainstream browsers (e.g. HEVC).
* The client renders the fallback card directly, no flash.
* - `true` — confidently decodable (web codec in a web container).
* - `null` — unknown / optimistic. The codec couldn't be probed (ffmpeg
* absent or probe failed), or it's a web-safe codec in a
* container with inconsistent cross-browser support (e.g. H.264
* in .mov). The client attempts playback and degrades to the
* fallback via the `<video>` `onError` handler.
*
* We never widen `false` beyond codecs we are certain fail everywhere, so an
* instance without ffmpeg (codec always undefined) keeps today's behaviour
* (attempt playback) rather than regressing every video to "unplayable".
*/
/** ffprobe `codec_name` (or mp4/mov box tag) values that no mainstream browser decodes. */
const UNPLAYABLE_VIDEO_CODECS = new Set([
// HEVC / H.265 — codec_name is `hevc`; box tags are `hvc1` / `hev1`.
'hevc', 'hvc1', 'hev1', 'h265',
// Apple ProRes — editing/intermediate codec, never web-decodable.
'prores',
// Windows Media Video.
'wmv1', 'wmv2', 'wmv3', 'vc1',
// Legacy / capture codecs.
'mpeg1video', 'mpeg2video', 'dnxhd', 'mjpeg', 'vp6', 'vp6f',
]);
/** Codecs every modern browser can decode when in a web-standard container. */
const PLAYABLE_VIDEO_CODECS = new Set([
'h264', 'avc1', // H.264 / AVC
'vp8', 'vp9',
'av1', 'av01',
'theora',
]);
/** Containers with reliable cross-browser `<video>` support. */
const PLAYABLE_VIDEO_CONTAINERS = new Set([
'video/mp4',
'video/webm',
'video/ogg',
]);
/**
* Classify whether a video attachment can be played inline in a browser
* `<video>` element, given its container mimetype and the probed video codec.
*
* @param mimetype Container mimetype (e.g. `video/quicktime`).
* @param codec ffprobe `codec_name` of the primary video stream, if known.
* @returns `false` (known-unplayable), `true` (known-playable) or `null` (unknown/optimistic).
*/
export function classifyVideoPlayable(
mimetype: string,
codec: string | null | undefined,
): boolean | null {
if (!mimetype.startsWith('video/')) return null;
if (!codec) return null;
const c = codec.toLowerCase();
if (UNPLAYABLE_VIDEO_CODECS.has(c)) return false;
if (PLAYABLE_VIDEO_CONTAINERS.has(mimetype) && PLAYABLE_VIDEO_CODECS.has(c)) return true;
// Web-safe codec in a shaky container (H.264 in .mov), or an unrecognised
// codec we can't vouch for: stay optimistic and let the client's onError
// handler catch a genuine playback failure.
return null;
}
+10 -5
View File
@@ -138,19 +138,21 @@ export async function probeImageDimensions(filepath: string): Promise<{ width: n
export async function probeMediaMeta(
filepath: string,
mimetype: string,
): Promise<{ width?: number; height?: number; duration?: number } | null> {
): Promise<{ width?: number; height?: number; duration?: number; codec?: string } | null> {
if (!(await checkFfmpeg())) return null;
try {
const result: { width?: number; height?: number; duration?: number } = {};
const result: { width?: number; height?: number; duration?: number; codec?: string } = {};
// Video dimensions (stream-level)
// Video dimensions + codec (stream-level). The codec drives web-playability
// classification (see utils/mediaPlayable.ts) so the client can render a
// download fallback for formats the browser can't decode (e.g. HEVC .mov).
if (mimetype.startsWith('video/')) {
try {
const { stdout } = await execFile('ffprobe', [
'-v', 'error',
'-select_streams', 'v:0',
'-show_entries', 'stream=width,height',
'-show_entries', 'stream=width,height,codec_name',
'-of', 'json',
'-i', filepath,
], { timeout: FFMPEG_TIMEOUT });
@@ -161,7 +163,10 @@ export async function probeMediaMeta(
result.width = stream.width;
result.height = stream.height;
}
} catch { /* dimension probe failed — continue for duration */ }
if (typeof stream?.codec_name === 'string' && stream.codec_name) {
result.codec = stream.codec_name;
}
} catch { /* dimension/codec probe failed — continue for duration */ }
}
// Duration (format-level — works for video and audio)