From e7a18bd28c807a731b5c36139bf176d2f330a2d6 Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Mon, 23 Mar 2026 22:10:16 +0100 Subject: [PATCH] fix: screenshare codec fallback, native FPS, and connection info - Gaming mode uses H.264 primary (hardware NVENC encoding, zero CPU impact on games). Text mode uses VP9 primary with H.264 backup and SIMULCAST policy for Safari compatibility. - Fix native mode starting at 30fps by decoupling frameRate constraint from resolution constraint in both screenShare.ts overdrive and useLiveKit.ts updateActiveTracks. - Filter paused backup codec tracks in Connection Info stats so dead 0kbps entries don't show alongside active codec tracks. --- packages/web/src/hooks/useLiveKit.ts | 7 +++-- packages/web/src/hooks/useTrackStats.ts | 20 ++++++++++-- packages/web/src/utils/screenShare.ts | 42 ++++++++++++++++++++----- 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/packages/web/src/hooks/useLiveKit.ts b/packages/web/src/hooks/useLiveKit.ts index 8dd02b8b..1c2c80e7 100644 --- a/packages/web/src/hooks/useLiveKit.ts +++ b/packages/web/src/hooks/useLiveKit.ts @@ -612,9 +612,12 @@ export function useLiveKit() { if (screenPub?.videoTrack) { const mediaTrack = getMediaStreamTrack(screenPub.videoTrack); if (mediaTrack) { - // Skip resolution constraints for native mode — capture is already at native dims if (opts.capture.width > 0 && opts.capture.height > 0) { - await mediaTrack.applyConstraints({ width: { ideal: opts.capture.width }, height: { ideal: opts.capture.height }, frameRate: { ideal: opts.capture.frameRate } }); + // Standard mode: apply resolution + frameRate together + await mediaTrack.applyConstraints({ width: { ideal: opts.capture.width }, height: { ideal: opts.capture.height }, frameRate: { ideal: opts.capture.frameRate, min: 15 } }); + } else { + // Native mode: apply frameRate only — never pass 0 to width/height + await mediaTrack.applyConstraints({ frameRate: { ideal: opts.capture.frameRate, min: 15 } }); } mediaTrack.contentHint = opts.contentHint; } diff --git a/packages/web/src/hooks/useTrackStats.ts b/packages/web/src/hooks/useTrackStats.ts index 55ff42cf..d37f9b7b 100644 --- a/packages/web/src/hooks/useTrackStats.ts +++ b/packages/web/src/hooks/useTrackStats.ts @@ -511,14 +511,30 @@ export function useTrackStats(enabled: boolean): TrackStatsSnapshot | null { network.packetLoss = (totalPacketsLost / totalPackets) * 100; } - // ── Step F: Cleanup stale prevSample entries ── + // ── Step F: Filter paused backup codec tracks ── + // With backupCodec enabled, the publisher may have two concurrent outbound + // video tracks for the same source (e.g. VP9 + H.264). When one is paused + // by dynacast (0 bitrate), hide it if an active sibling exists. + const filteredVideoTracks = videoTracks.filter((track) => { + if (track.direction !== 'send' || track.bitrate > 0) return true; + const hasActiveSibling = videoTracks.some( + (other) => + other !== track && + other.direction === 'send' && + other.source === track.source && + other.bitrate > 0, + ); + return !hasActiveSibling; + }); + + // ── Step G: Cleanup stale prevSample entries ── for (const key of prev.keys()) { if (!seenKeys.has(key)) { prev.delete(key); } } - setSnapshot({ network, audioTracks, videoTracks }); + setSnapshot({ network, audioTracks, videoTracks: filteredVideoTracks }); }; poll(); diff --git a/packages/web/src/utils/screenShare.ts b/packages/web/src/utils/screenShare.ts index 29b5440d..46d6ce75 100644 --- a/packages/web/src/utils/screenShare.ts +++ b/packages/web/src/utils/screenShare.ts @@ -1,4 +1,4 @@ -import { Room, Track } from 'livekit-client'; +import { Room, Track, BackupCodecPolicy } from 'livekit-client'; import { useVoiceStore } from '../stores/voiceStore'; import type { ScreenShareConfig } from '../stores/voiceStore'; import { getStreamingLimits } from '../stores/settingsStore'; @@ -23,7 +23,13 @@ export interface OverdriveOptions { export interface ScreenShareBuildResult { capture: { width: number; height: number; frameRate: number }; - publish: { videoCodec: 'vp9'; videoEncoding: { maxBitrate: number; maxFramerate: number }; simulcast: false }; + publish: { + videoCodec: 'vp9' | 'h264'; + videoEncoding: { maxBitrate: number; maxFramerate: number }; + simulcast: false; + backupCodec?: { codec: 'h264'; encoding: { maxBitrate: number; maxFramerate: number } }; + backupCodecPolicy?: BackupCodecPolicy; + }; overdrive: OverdriveOptions; contentHint: 'motion' | 'detail'; } @@ -125,20 +131,33 @@ export function buildScreenShareOptions(config: ScreenShareConfig): ScreenShareB const bps = clampedKbps * 1000; const minBps = Math.round(bps * 0.25); + // Gaming mode: H.264 primary (hardware NVENC/QSV encoding, zero CPU impact on games) + // Text mode: VP9 primary (better compression for sharp text) with H.264 backup for Safari + const isGaming = mode === 'gaming'; + return { capture: { width: captureWidth, height: captureHeight, frameRate: fps }, publish: { - videoCodec: 'vp9', + videoCodec: isGaming ? 'h264' : 'vp9', videoEncoding: { maxBitrate: bps, maxFramerate: fps }, simulcast: false, + // H.264 is universally supported — no backup needed for gaming mode. + // VP9 needs H.264 backup for Safari/incompatible viewers. + ...(isGaming ? {} : { + backupCodec: { + codec: 'h264' as const, + encoding: { maxBitrate: bps, maxFramerate: fps }, + }, + backupCodecPolicy: BackupCodecPolicy.SIMULCAST, + }), }, overdrive: { maxBitrate: bps, maxFramerate: fps, minBitrate: minBps, - degradationPreference: mode === 'text' ? 'maintain-resolution' : 'balanced', + degradationPreference: isGaming ? 'balanced' : 'maintain-resolution', }, - contentHint: mode === 'text' ? 'detail' : 'motion', + contentHint: isGaming ? 'motion' : 'detail', }; } @@ -243,7 +262,11 @@ export async function startScreenShare(room: Room): Promise { videoCodec: opts.publish.videoCodec, videoEncoding: opts.publish.videoEncoding, simulcast: opts.publish.simulcast, - } as any); + ...(opts.publish.backupCodec ? { + backupCodec: opts.publish.backupCodec, + backupCodecPolicy: opts.publish.backupCodecPolicy, + } : {}), + }); console.log('[SS] setScreenShareEnabled returned:', !!track); if (!track) { @@ -279,13 +302,18 @@ function applyScreenShareOverdrive(room: Room): void { const screenPub = room.localParticipant.getTrackPublications() .find(p => p.source === Track.Source.ScreenShare); if (screenPub?.track?.mediaStreamTrack) { - // Skip resolution constraints for native mode — capture is already at native dims if (freshOpts.capture.width > 0 && freshOpts.capture.height > 0) { + // Standard mode: apply resolution + frameRate together await screenPub.track.mediaStreamTrack.applyConstraints({ width: { ideal: freshOpts.capture.width }, height: { ideal: freshOpts.capture.height }, frameRate: { ideal: freshOpts.capture.frameRate, min: 15 }, }); + } else { + // Native mode: apply frameRate only — never pass 0 to width/height + await screenPub.track.mediaStreamTrack.applyConstraints({ + frameRate: { ideal: freshOpts.capture.frameRate, min: 15 }, + }); } screenPub.track.mediaStreamTrack.contentHint = freshOpts.contentHint;