fix: screenshare codec fallback, native FPS, and connection info
- Gaming mode uses H.264 primary (hardware NVENC encoding, zero CPU impact on games). Text mode uses VP9 primary with H.264 backup and SIMULCAST policy for Safari compatibility. - Fix native mode starting at 30fps by decoupling frameRate constraint from resolution constraint in both screenShare.ts overdrive and useLiveKit.ts updateActiveTracks. - Filter paused backup codec tracks in Connection Info stats so dead 0kbps entries don't show alongside active codec tracks.
This commit is contained in:
@@ -612,9 +612,12 @@ export function useLiveKit() {
|
|||||||
if (screenPub?.videoTrack) {
|
if (screenPub?.videoTrack) {
|
||||||
const mediaTrack = getMediaStreamTrack(screenPub.videoTrack);
|
const mediaTrack = getMediaStreamTrack(screenPub.videoTrack);
|
||||||
if (mediaTrack) {
|
if (mediaTrack) {
|
||||||
// Skip resolution constraints for native mode — capture is already at native dims
|
|
||||||
if (opts.capture.width > 0 && opts.capture.height > 0) {
|
if (opts.capture.width > 0 && opts.capture.height > 0) {
|
||||||
await mediaTrack.applyConstraints({ width: { ideal: opts.capture.width }, height: { ideal: opts.capture.height }, frameRate: { ideal: opts.capture.frameRate } });
|
// Standard mode: apply resolution + frameRate together
|
||||||
|
await mediaTrack.applyConstraints({ width: { ideal: opts.capture.width }, height: { ideal: opts.capture.height }, frameRate: { ideal: opts.capture.frameRate, min: 15 } });
|
||||||
|
} else {
|
||||||
|
// Native mode: apply frameRate only — never pass 0 to width/height
|
||||||
|
await mediaTrack.applyConstraints({ frameRate: { ideal: opts.capture.frameRate, min: 15 } });
|
||||||
}
|
}
|
||||||
mediaTrack.contentHint = opts.contentHint;
|
mediaTrack.contentHint = opts.contentHint;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -511,14 +511,30 @@ export function useTrackStats(enabled: boolean): TrackStatsSnapshot | null {
|
|||||||
network.packetLoss = (totalPacketsLost / totalPackets) * 100;
|
network.packetLoss = (totalPacketsLost / totalPackets) * 100;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Step F: Cleanup stale prevSample entries ──
|
// ── Step F: Filter paused backup codec tracks ──
|
||||||
|
// With backupCodec enabled, the publisher may have two concurrent outbound
|
||||||
|
// video tracks for the same source (e.g. VP9 + H.264). When one is paused
|
||||||
|
// by dynacast (0 bitrate), hide it if an active sibling exists.
|
||||||
|
const filteredVideoTracks = videoTracks.filter((track) => {
|
||||||
|
if (track.direction !== 'send' || track.bitrate > 0) return true;
|
||||||
|
const hasActiveSibling = videoTracks.some(
|
||||||
|
(other) =>
|
||||||
|
other !== track &&
|
||||||
|
other.direction === 'send' &&
|
||||||
|
other.source === track.source &&
|
||||||
|
other.bitrate > 0,
|
||||||
|
);
|
||||||
|
return !hasActiveSibling;
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Step G: Cleanup stale prevSample entries ──
|
||||||
for (const key of prev.keys()) {
|
for (const key of prev.keys()) {
|
||||||
if (!seenKeys.has(key)) {
|
if (!seenKeys.has(key)) {
|
||||||
prev.delete(key);
|
prev.delete(key);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
setSnapshot({ network, audioTracks, videoTracks });
|
setSnapshot({ network, audioTracks, videoTracks: filteredVideoTracks });
|
||||||
};
|
};
|
||||||
|
|
||||||
poll();
|
poll();
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { Room, Track } from 'livekit-client';
|
import { Room, Track, BackupCodecPolicy } from 'livekit-client';
|
||||||
import { useVoiceStore } from '../stores/voiceStore';
|
import { useVoiceStore } from '../stores/voiceStore';
|
||||||
import type { ScreenShareConfig } from '../stores/voiceStore';
|
import type { ScreenShareConfig } from '../stores/voiceStore';
|
||||||
import { getStreamingLimits } from '../stores/settingsStore';
|
import { getStreamingLimits } from '../stores/settingsStore';
|
||||||
@@ -23,7 +23,13 @@ export interface OverdriveOptions {
|
|||||||
|
|
||||||
export interface ScreenShareBuildResult {
|
export interface ScreenShareBuildResult {
|
||||||
capture: { width: number; height: number; frameRate: number };
|
capture: { width: number; height: number; frameRate: number };
|
||||||
publish: { videoCodec: 'vp9'; videoEncoding: { maxBitrate: number; maxFramerate: number }; simulcast: false };
|
publish: {
|
||||||
|
videoCodec: 'vp9' | 'h264';
|
||||||
|
videoEncoding: { maxBitrate: number; maxFramerate: number };
|
||||||
|
simulcast: false;
|
||||||
|
backupCodec?: { codec: 'h264'; encoding: { maxBitrate: number; maxFramerate: number } };
|
||||||
|
backupCodecPolicy?: BackupCodecPolicy;
|
||||||
|
};
|
||||||
overdrive: OverdriveOptions;
|
overdrive: OverdriveOptions;
|
||||||
contentHint: 'motion' | 'detail';
|
contentHint: 'motion' | 'detail';
|
||||||
}
|
}
|
||||||
@@ -125,20 +131,33 @@ export function buildScreenShareOptions(config: ScreenShareConfig): ScreenShareB
|
|||||||
const bps = clampedKbps * 1000;
|
const bps = clampedKbps * 1000;
|
||||||
const minBps = Math.round(bps * 0.25);
|
const minBps = Math.round(bps * 0.25);
|
||||||
|
|
||||||
|
// Gaming mode: H.264 primary (hardware NVENC/QSV encoding, zero CPU impact on games)
|
||||||
|
// Text mode: VP9 primary (better compression for sharp text) with H.264 backup for Safari
|
||||||
|
const isGaming = mode === 'gaming';
|
||||||
|
|
||||||
return {
|
return {
|
||||||
capture: { width: captureWidth, height: captureHeight, frameRate: fps },
|
capture: { width: captureWidth, height: captureHeight, frameRate: fps },
|
||||||
publish: {
|
publish: {
|
||||||
videoCodec: 'vp9',
|
videoCodec: isGaming ? 'h264' : 'vp9',
|
||||||
videoEncoding: { maxBitrate: bps, maxFramerate: fps },
|
videoEncoding: { maxBitrate: bps, maxFramerate: fps },
|
||||||
simulcast: false,
|
simulcast: false,
|
||||||
|
// H.264 is universally supported — no backup needed for gaming mode.
|
||||||
|
// VP9 needs H.264 backup for Safari/incompatible viewers.
|
||||||
|
...(isGaming ? {} : {
|
||||||
|
backupCodec: {
|
||||||
|
codec: 'h264' as const,
|
||||||
|
encoding: { maxBitrate: bps, maxFramerate: fps },
|
||||||
|
},
|
||||||
|
backupCodecPolicy: BackupCodecPolicy.SIMULCAST,
|
||||||
|
}),
|
||||||
},
|
},
|
||||||
overdrive: {
|
overdrive: {
|
||||||
maxBitrate: bps,
|
maxBitrate: bps,
|
||||||
maxFramerate: fps,
|
maxFramerate: fps,
|
||||||
minBitrate: minBps,
|
minBitrate: minBps,
|
||||||
degradationPreference: mode === 'text' ? 'maintain-resolution' : 'balanced',
|
degradationPreference: isGaming ? 'balanced' : 'maintain-resolution',
|
||||||
},
|
},
|
||||||
contentHint: mode === 'text' ? 'detail' : 'motion',
|
contentHint: isGaming ? 'motion' : 'detail',
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -243,7 +262,11 @@ export async function startScreenShare(room: Room): Promise<boolean> {
|
|||||||
videoCodec: opts.publish.videoCodec,
|
videoCodec: opts.publish.videoCodec,
|
||||||
videoEncoding: opts.publish.videoEncoding,
|
videoEncoding: opts.publish.videoEncoding,
|
||||||
simulcast: opts.publish.simulcast,
|
simulcast: opts.publish.simulcast,
|
||||||
} as any);
|
...(opts.publish.backupCodec ? {
|
||||||
|
backupCodec: opts.publish.backupCodec,
|
||||||
|
backupCodecPolicy: opts.publish.backupCodecPolicy,
|
||||||
|
} : {}),
|
||||||
|
});
|
||||||
|
|
||||||
console.log('[SS] setScreenShareEnabled returned:', !!track);
|
console.log('[SS] setScreenShareEnabled returned:', !!track);
|
||||||
if (!track) {
|
if (!track) {
|
||||||
@@ -279,13 +302,18 @@ function applyScreenShareOverdrive(room: Room): void {
|
|||||||
const screenPub = room.localParticipant.getTrackPublications()
|
const screenPub = room.localParticipant.getTrackPublications()
|
||||||
.find(p => p.source === Track.Source.ScreenShare);
|
.find(p => p.source === Track.Source.ScreenShare);
|
||||||
if (screenPub?.track?.mediaStreamTrack) {
|
if (screenPub?.track?.mediaStreamTrack) {
|
||||||
// Skip resolution constraints for native mode — capture is already at native dims
|
|
||||||
if (freshOpts.capture.width > 0 && freshOpts.capture.height > 0) {
|
if (freshOpts.capture.width > 0 && freshOpts.capture.height > 0) {
|
||||||
|
// Standard mode: apply resolution + frameRate together
|
||||||
await screenPub.track.mediaStreamTrack.applyConstraints({
|
await screenPub.track.mediaStreamTrack.applyConstraints({
|
||||||
width: { ideal: freshOpts.capture.width },
|
width: { ideal: freshOpts.capture.width },
|
||||||
height: { ideal: freshOpts.capture.height },
|
height: { ideal: freshOpts.capture.height },
|
||||||
frameRate: { ideal: freshOpts.capture.frameRate, min: 15 },
|
frameRate: { ideal: freshOpts.capture.frameRate, min: 15 },
|
||||||
});
|
});
|
||||||
|
} else {
|
||||||
|
// Native mode: apply frameRate only — never pass 0 to width/height
|
||||||
|
await screenPub.track.mediaStreamTrack.applyConstraints({
|
||||||
|
frameRate: { ideal: freshOpts.capture.frameRate, min: 15 },
|
||||||
|
});
|
||||||
}
|
}
|
||||||
screenPub.track.mediaStreamTrack.contentHint = freshOpts.contentHint;
|
screenPub.track.mediaStreamTrack.contentHint = freshOpts.contentHint;
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user