fix(uploads): graceful fallback for browser-unplayable video (HEVC .mov)
macOS screen recordings are HEVC inside a .mov container, which Chromium, Firefox and stock Electron can't decode. The file uploaded fine and a server-side ffmpeg poster was generated, but inline <video> playback failed silently — stuck at 0:00 with no error, since AttachmentRenderer had no error handling. Root cause: the system had no concept of web-playability. Server detects, client degrades: - mediaPlayable.ts: classifyVideoPlayable(mimetype, codec) — tri-state (false = known-undecodable e.g. HEVC/ProRes, true = web codec in web container, null = unknown/optimistic). Never widens `false` beyond codecs that fail everywhere, so ffmpeg-less instances keep prior behaviour. - probeMediaMeta now captures the video codec_name; the upload finish hook stores the verdict in the new attachments.playable column (migration 0007). - Flag propagated through every serializer: space messages, DMs, WS, and federation relay (outbound + inbound) — federation-compatible. - VideoAttachment component: playable===false renders a download card (poster + "Can't play here — download" + name/duration/size) with no dead-player flash; otherwise plays inline with an onError fallback to the same card. Specs updated: uploads.md, database.md, federation.md.
This commit is contained in:
@@ -446,6 +446,7 @@ export function queueDmRelay(
|
||||
width: a.width ?? undefined,
|
||||
height: a.height ?? undefined,
|
||||
duration: a.duration ?? undefined,
|
||||
playable: a.playable ?? null,
|
||||
thumbnailFilename: a.thumbnailFilename ?? undefined,
|
||||
sourceUrl: `${domainOrigin}/api/uploads/${a.filename}`,
|
||||
}));
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { classifyVideoPlayable } from './mediaPlayable';
|
||||
|
||||
describe('classifyVideoPlayable', () => {
|
||||
it('marks HEVC as not web-playable regardless of container', () => {
|
||||
// macOS screen recordings: video/quicktime + hevc — the reported bug.
|
||||
expect(classifyVideoPlayable('video/quicktime', 'hevc')).toBe(false);
|
||||
// HEVC inside an mp4 container is equally undecodable in Chromium/Firefox.
|
||||
expect(classifyVideoPlayable('video/mp4', 'hevc')).toBe(false);
|
||||
// ffprobe sometimes reports the tag rather than the codec name.
|
||||
expect(classifyVideoPlayable('video/quicktime', 'hvc1')).toBe(false);
|
||||
expect(classifyVideoPlayable('video/quicktime', 'hev1')).toBe(false);
|
||||
});
|
||||
|
||||
it('marks other known-undecodable codecs as not web-playable', () => {
|
||||
expect(classifyVideoPlayable('video/quicktime', 'prores')).toBe(false);
|
||||
expect(classifyVideoPlayable('video/x-msvideo', 'wmv3')).toBe(false);
|
||||
});
|
||||
|
||||
it('marks web-standard codec in a web container as playable', () => {
|
||||
expect(classifyVideoPlayable('video/mp4', 'h264')).toBe(true);
|
||||
expect(classifyVideoPlayable('video/webm', 'vp9')).toBe(true);
|
||||
expect(classifyVideoPlayable('video/webm', 'av1')).toBe(true);
|
||||
// ffprobe reports H.264 as 'h264'; the mp4 box tag is 'avc1'.
|
||||
expect(classifyVideoPlayable('video/mp4', 'avc1')).toBe(true);
|
||||
});
|
||||
|
||||
it('is optimistic (null) for web-safe codec in a non-web container', () => {
|
||||
// H.264 in a .mov plays in Chrome/Safari but not Firefox — let the client
|
||||
// attempt playback and fall back via onError rather than blocking it.
|
||||
expect(classifyVideoPlayable('video/quicktime', 'h264')).toBeNull();
|
||||
});
|
||||
|
||||
it('is optimistic (null) when the codec is unknown', () => {
|
||||
// ffmpeg unavailable / probe failed — must not regress to "unplayable".
|
||||
expect(classifyVideoPlayable('video/mp4', undefined)).toBeNull();
|
||||
expect(classifyVideoPlayable('video/mp4', null)).toBeNull();
|
||||
expect(classifyVideoPlayable('video/mp4', '')).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for non-video mimetypes', () => {
|
||||
expect(classifyVideoPlayable('audio/mpeg', 'mp3')).toBeNull();
|
||||
expect(classifyVideoPlayable('image/png', undefined)).toBeNull();
|
||||
});
|
||||
|
||||
it('is case-insensitive on the codec name', () => {
|
||||
expect(classifyVideoPlayable('video/quicktime', 'HEVC')).toBe(false);
|
||||
expect(classifyVideoPlayable('video/mp4', 'H264')).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
/**
|
||||
* Web-playability classification for video attachments.
|
||||
*
|
||||
* The browser `<video>` element can only decode a subset of the formats users
|
||||
* upload. The dominant failure case is a macOS screen recording — a
|
||||
* `video/quicktime` (.mov) container holding an HEVC (H.265) stream — which
|
||||
* Chromium, Firefox and stock Electron cannot decode. The file uploads fine,
|
||||
* a server-side ffmpeg poster is generated, but inline playback silently fails
|
||||
* (stuck at 0:00 with no error). We persist this classification per attachment
|
||||
* so the client can render a download fallback instead of a dead player.
|
||||
*
|
||||
* The result is a deliberate tri-state:
|
||||
* - `false` — confidently undecodable in mainstream browsers (e.g. HEVC).
|
||||
* The client renders the fallback card directly, no flash.
|
||||
* - `true` — confidently decodable (web codec in a web container).
|
||||
* - `null` — unknown / optimistic. The codec couldn't be probed (ffmpeg
|
||||
* absent or probe failed), or it's a web-safe codec in a
|
||||
* container with inconsistent cross-browser support (e.g. H.264
|
||||
* in .mov). The client attempts playback and degrades to the
|
||||
* fallback via the `<video>` `onError` handler.
|
||||
*
|
||||
* We never widen `false` beyond codecs we are certain fail everywhere, so an
|
||||
* instance without ffmpeg (codec always undefined) keeps today's behaviour
|
||||
* (attempt playback) rather than regressing every video to "unplayable".
|
||||
*/
|
||||
|
||||
/** ffprobe `codec_name` (or mp4/mov box tag) values that no mainstream browser decodes. */
|
||||
const UNPLAYABLE_VIDEO_CODECS = new Set([
|
||||
// HEVC / H.265 — codec_name is `hevc`; box tags are `hvc1` / `hev1`.
|
||||
'hevc', 'hvc1', 'hev1', 'h265',
|
||||
// Apple ProRes — editing/intermediate codec, never web-decodable.
|
||||
'prores',
|
||||
// Windows Media Video.
|
||||
'wmv1', 'wmv2', 'wmv3', 'vc1',
|
||||
// Legacy / capture codecs.
|
||||
'mpeg1video', 'mpeg2video', 'dnxhd', 'mjpeg', 'vp6', 'vp6f',
|
||||
]);
|
||||
|
||||
/** Codecs every modern browser can decode when in a web-standard container. */
|
||||
const PLAYABLE_VIDEO_CODECS = new Set([
|
||||
'h264', 'avc1', // H.264 / AVC
|
||||
'vp8', 'vp9',
|
||||
'av1', 'av01',
|
||||
'theora',
|
||||
]);
|
||||
|
||||
/** Containers with reliable cross-browser `<video>` support. */
|
||||
const PLAYABLE_VIDEO_CONTAINERS = new Set([
|
||||
'video/mp4',
|
||||
'video/webm',
|
||||
'video/ogg',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Classify whether a video attachment can be played inline in a browser
|
||||
* `<video>` element, given its container mimetype and the probed video codec.
|
||||
*
|
||||
* @param mimetype Container mimetype (e.g. `video/quicktime`).
|
||||
* @param codec ffprobe `codec_name` of the primary video stream, if known.
|
||||
* @returns `false` (known-unplayable), `true` (known-playable) or `null` (unknown/optimistic).
|
||||
*/
|
||||
export function classifyVideoPlayable(
|
||||
mimetype: string,
|
||||
codec: string | null | undefined,
|
||||
): boolean | null {
|
||||
if (!mimetype.startsWith('video/')) return null;
|
||||
if (!codec) return null;
|
||||
|
||||
const c = codec.toLowerCase();
|
||||
if (UNPLAYABLE_VIDEO_CODECS.has(c)) return false;
|
||||
if (PLAYABLE_VIDEO_CONTAINERS.has(mimetype) && PLAYABLE_VIDEO_CODECS.has(c)) return true;
|
||||
|
||||
// Web-safe codec in a shaky container (H.264 in .mov), or an unrecognised
|
||||
// codec we can't vouch for: stay optimistic and let the client's onError
|
||||
// handler catch a genuine playback failure.
|
||||
return null;
|
||||
}
|
||||
@@ -138,19 +138,21 @@ export async function probeImageDimensions(filepath: string): Promise<{ width: n
|
||||
export async function probeMediaMeta(
|
||||
filepath: string,
|
||||
mimetype: string,
|
||||
): Promise<{ width?: number; height?: number; duration?: number } | null> {
|
||||
): Promise<{ width?: number; height?: number; duration?: number; codec?: string } | null> {
|
||||
if (!(await checkFfmpeg())) return null;
|
||||
|
||||
try {
|
||||
const result: { width?: number; height?: number; duration?: number } = {};
|
||||
const result: { width?: number; height?: number; duration?: number; codec?: string } = {};
|
||||
|
||||
// Video dimensions (stream-level)
|
||||
// Video dimensions + codec (stream-level). The codec drives web-playability
|
||||
// classification (see utils/mediaPlayable.ts) so the client can render a
|
||||
// download fallback for formats the browser can't decode (e.g. HEVC .mov).
|
||||
if (mimetype.startsWith('video/')) {
|
||||
try {
|
||||
const { stdout } = await execFile('ffprobe', [
|
||||
'-v', 'error',
|
||||
'-select_streams', 'v:0',
|
||||
'-show_entries', 'stream=width,height',
|
||||
'-show_entries', 'stream=width,height,codec_name',
|
||||
'-of', 'json',
|
||||
'-i', filepath,
|
||||
], { timeout: FFMPEG_TIMEOUT });
|
||||
@@ -161,7 +163,10 @@ export async function probeMediaMeta(
|
||||
result.width = stream.width;
|
||||
result.height = stream.height;
|
||||
}
|
||||
} catch { /* dimension probe failed — continue for duration */ }
|
||||
if (typeof stream?.codec_name === 'string' && stream.codec_name) {
|
||||
result.codec = stream.codec_name;
|
||||
}
|
||||
} catch { /* dimension/codec probe failed — continue for duration */ }
|
||||
}
|
||||
|
||||
// Duration (format-level — works for video and audio)
|
||||
|
||||
Reference in New Issue
Block a user