fix: screenshare codec fallback, native FPS, and connection info

- Gaming mode uses H.264 primary (hardware NVENC encoding, zero CPU
  impact on games). Text mode uses VP9 primary with H.264 backup
  and SIMULCAST policy for Safari compatibility.
- Fix native mode starting at 30fps by decoupling frameRate constraint
  from resolution constraint in both screenShare.ts overdrive and
  useLiveKit.ts updateActiveTracks.
- Filter paused backup codec tracks in Connection Info stats so dead
  0kbps entries don't show alongside active codec tracks.
This commit is contained in:
Jannis Braun
2026-03-23 22:10:16 +01:00
parent 7693cc737e
commit e7a18bd28c
3 changed files with 58 additions and 11 deletions
+5 -2
View File
@@ -612,9 +612,12 @@ export function useLiveKit() {
if (screenPub?.videoTrack) { if (screenPub?.videoTrack) {
const mediaTrack = getMediaStreamTrack(screenPub.videoTrack); const mediaTrack = getMediaStreamTrack(screenPub.videoTrack);
if (mediaTrack) { if (mediaTrack) {
// Skip resolution constraints for native mode — capture is already at native dims
if (opts.capture.width > 0 && opts.capture.height > 0) { if (opts.capture.width > 0 && opts.capture.height > 0) {
await mediaTrack.applyConstraints({ width: { ideal: opts.capture.width }, height: { ideal: opts.capture.height }, frameRate: { ideal: opts.capture.frameRate } }); // Standard mode: apply resolution + frameRate together
await mediaTrack.applyConstraints({ width: { ideal: opts.capture.width }, height: { ideal: opts.capture.height }, frameRate: { ideal: opts.capture.frameRate, min: 15 } });
} else {
// Native mode: apply frameRate only — never pass 0 to width/height
await mediaTrack.applyConstraints({ frameRate: { ideal: opts.capture.frameRate, min: 15 } });
} }
mediaTrack.contentHint = opts.contentHint; mediaTrack.contentHint = opts.contentHint;
} }
+18 -2
View File
@@ -511,14 +511,30 @@ export function useTrackStats(enabled: boolean): TrackStatsSnapshot | null {
network.packetLoss = (totalPacketsLost / totalPackets) * 100; network.packetLoss = (totalPacketsLost / totalPackets) * 100;
} }
// ── Step F: Cleanup stale prevSample entries ── // ── Step F: Filter paused backup codec tracks ──
// With backupCodec enabled, the publisher may have two concurrent outbound
// video tracks for the same source (e.g. VP9 + H.264). When one is paused
// by dynacast (0 bitrate), hide it if an active sibling exists.
const filteredVideoTracks = videoTracks.filter((track) => {
if (track.direction !== 'send' || track.bitrate > 0) return true;
const hasActiveSibling = videoTracks.some(
(other) =>
other !== track &&
other.direction === 'send' &&
other.source === track.source &&
other.bitrate > 0,
);
return !hasActiveSibling;
});
// ── Step G: Cleanup stale prevSample entries ──
for (const key of prev.keys()) { for (const key of prev.keys()) {
if (!seenKeys.has(key)) { if (!seenKeys.has(key)) {
prev.delete(key); prev.delete(key);
} }
} }
setSnapshot({ network, audioTracks, videoTracks }); setSnapshot({ network, audioTracks, videoTracks: filteredVideoTracks });
}; };
poll(); poll();
+35 -7
View File
@@ -1,4 +1,4 @@
import { Room, Track } from 'livekit-client'; import { Room, Track, BackupCodecPolicy } from 'livekit-client';
import { useVoiceStore } from '../stores/voiceStore'; import { useVoiceStore } from '../stores/voiceStore';
import type { ScreenShareConfig } from '../stores/voiceStore'; import type { ScreenShareConfig } from '../stores/voiceStore';
import { getStreamingLimits } from '../stores/settingsStore'; import { getStreamingLimits } from '../stores/settingsStore';
@@ -23,7 +23,13 @@ export interface OverdriveOptions {
export interface ScreenShareBuildResult { export interface ScreenShareBuildResult {
capture: { width: number; height: number; frameRate: number }; capture: { width: number; height: number; frameRate: number };
publish: { videoCodec: 'vp9'; videoEncoding: { maxBitrate: number; maxFramerate: number }; simulcast: false }; publish: {
videoCodec: 'vp9' | 'h264';
videoEncoding: { maxBitrate: number; maxFramerate: number };
simulcast: false;
backupCodec?: { codec: 'h264'; encoding: { maxBitrate: number; maxFramerate: number } };
backupCodecPolicy?: BackupCodecPolicy;
};
overdrive: OverdriveOptions; overdrive: OverdriveOptions;
contentHint: 'motion' | 'detail'; contentHint: 'motion' | 'detail';
} }
@@ -125,20 +131,33 @@ export function buildScreenShareOptions(config: ScreenShareConfig): ScreenShareB
const bps = clampedKbps * 1000; const bps = clampedKbps * 1000;
const minBps = Math.round(bps * 0.25); const minBps = Math.round(bps * 0.25);
// Gaming mode: H.264 primary (hardware NVENC/QSV encoding, zero CPU impact on games)
// Text mode: VP9 primary (better compression for sharp text) with H.264 backup for Safari
const isGaming = mode === 'gaming';
return { return {
capture: { width: captureWidth, height: captureHeight, frameRate: fps }, capture: { width: captureWidth, height: captureHeight, frameRate: fps },
publish: { publish: {
videoCodec: 'vp9', videoCodec: isGaming ? 'h264' : 'vp9',
videoEncoding: { maxBitrate: bps, maxFramerate: fps }, videoEncoding: { maxBitrate: bps, maxFramerate: fps },
simulcast: false, simulcast: false,
// H.264 is universally supported — no backup needed for gaming mode.
// VP9 needs H.264 backup for Safari/incompatible viewers.
...(isGaming ? {} : {
backupCodec: {
codec: 'h264' as const,
encoding: { maxBitrate: bps, maxFramerate: fps },
},
backupCodecPolicy: BackupCodecPolicy.SIMULCAST,
}),
}, },
overdrive: { overdrive: {
maxBitrate: bps, maxBitrate: bps,
maxFramerate: fps, maxFramerate: fps,
minBitrate: minBps, minBitrate: minBps,
degradationPreference: mode === 'text' ? 'maintain-resolution' : 'balanced', degradationPreference: isGaming ? 'balanced' : 'maintain-resolution',
}, },
contentHint: mode === 'text' ? 'detail' : 'motion', contentHint: isGaming ? 'motion' : 'detail',
}; };
} }
@@ -243,7 +262,11 @@ export async function startScreenShare(room: Room): Promise<boolean> {
videoCodec: opts.publish.videoCodec, videoCodec: opts.publish.videoCodec,
videoEncoding: opts.publish.videoEncoding, videoEncoding: opts.publish.videoEncoding,
simulcast: opts.publish.simulcast, simulcast: opts.publish.simulcast,
} as any); ...(opts.publish.backupCodec ? {
backupCodec: opts.publish.backupCodec,
backupCodecPolicy: opts.publish.backupCodecPolicy,
} : {}),
});
console.log('[SS] setScreenShareEnabled returned:', !!track); console.log('[SS] setScreenShareEnabled returned:', !!track);
if (!track) { if (!track) {
@@ -279,13 +302,18 @@ function applyScreenShareOverdrive(room: Room): void {
const screenPub = room.localParticipant.getTrackPublications() const screenPub = room.localParticipant.getTrackPublications()
.find(p => p.source === Track.Source.ScreenShare); .find(p => p.source === Track.Source.ScreenShare);
if (screenPub?.track?.mediaStreamTrack) { if (screenPub?.track?.mediaStreamTrack) {
// Skip resolution constraints for native mode — capture is already at native dims
if (freshOpts.capture.width > 0 && freshOpts.capture.height > 0) { if (freshOpts.capture.width > 0 && freshOpts.capture.height > 0) {
// Standard mode: apply resolution + frameRate together
await screenPub.track.mediaStreamTrack.applyConstraints({ await screenPub.track.mediaStreamTrack.applyConstraints({
width: { ideal: freshOpts.capture.width }, width: { ideal: freshOpts.capture.width },
height: { ideal: freshOpts.capture.height }, height: { ideal: freshOpts.capture.height },
frameRate: { ideal: freshOpts.capture.frameRate, min: 15 }, frameRate: { ideal: freshOpts.capture.frameRate, min: 15 },
}); });
} else {
// Native mode: apply frameRate only — never pass 0 to width/height
await screenPub.track.mediaStreamTrack.applyConstraints({
frameRate: { ideal: freshOpts.capture.frameRate, min: 15 },
});
} }
screenPub.track.mediaStreamTrack.contentHint = freshOpts.contentHint; screenPub.track.mediaStreamTrack.contentHint = freshOpts.contentHint;