diff --git a/packages/web/src/components/voice/PictureInPicture.tsx b/packages/web/src/components/voice/PictureInPicture.tsx index 1a2e839a..43955f58 100644 --- a/packages/web/src/components/voice/PictureInPicture.tsx +++ b/packages/web/src/components/voice/PictureInPicture.tsx @@ -122,17 +122,26 @@ export function PictureInPicture() { return 'Call'; }, [currentVoiceChannelId, channels]); - // Video track attachment + // Derive the LiveKit Track from the selected stream's participant + const lkTrack = selectedStream + ? (selectedStream.type === 'screen' + ? selectedStream.participant.lkScreenTrack + : selectedStream.participant.lkVideoTrack) + : null; + + // Video track attachment — use LiveKit's track.attach() to register the element + // with the adaptive stream observer (enables SFU layer switching by viewport size) // shouldShow in deps ensures re-run when PiP becomes visible (videoRef was null before) useEffect(() => { const videoEl = videoRef.current; if (!videoEl) return; - if (selectedStream?.track) { - videoEl.srcObject = new MediaStream([selectedStream.track]); + if (lkTrack) { + lkTrack.attach(videoEl); + return () => { lkTrack.detach(videoEl); }; } else { videoEl.srcObject = null; } - }, [selectedStream?.track, shouldShow]); + }, [lkTrack, shouldShow]); // Initialize position to bottom-right useEffect(() => { diff --git a/packages/web/src/components/voice/StreamTile.tsx b/packages/web/src/components/voice/StreamTile.tsx index c679a500..b4236887 100644 --- a/packages/web/src/components/voice/StreamTile.tsx +++ b/packages/web/src/components/voice/StreamTile.tsx @@ -29,6 +29,7 @@ export function StreamTile({ tile, large }: StreamTileProps) { const isStreamMuted = streamMutes.get(userId) ?? false; const liveScreenTrack = tile.screenTrack?.readyState === 'live' ? tile.screenTrack : null; + const liveLkScreenTrack = liveScreenTrack ? tile.lkScreenTrack : null; // Quality badge state const [qualityBadge, setQualityBadge] = useState(''); @@ -37,16 +38,18 @@ export function StreamTile({ tile, large }: StreamTileProps) { const [contextMenu, setContextMenu] = useState<{ x: number; y: number } | null>(null); const [qualityPopoverOpen, setQualityPopoverOpen] = useState(false); - // --- VIDEO --- + // --- VIDEO --- use LiveKit's track.attach() to register the element + // with the adaptive stream observer (enables SFU layer switching by viewport size) useEffect(() => { const videoEl = videoRef.current; if (!videoEl) return; - if (liveScreenTrack) { - videoEl.srcObject = new MediaStream([liveScreenTrack]); + if (liveLkScreenTrack) { + liveLkScreenTrack.attach(videoEl); + return () => { liveLkScreenTrack.detach(videoEl); }; } else { videoEl.srcObject = null; } - }, [liveScreenTrack]); + }, [liveLkScreenTrack]); // Quality badge (poll every 3s) useEffect(() => { diff --git a/packages/web/src/components/voice/VoiceUser.tsx b/packages/web/src/components/voice/VoiceUser.tsx index cd022ecc..85a25f34 100644 --- a/packages/web/src/components/voice/VoiceUser.tsx +++ b/packages/web/src/components/voice/VoiceUser.tsx @@ -34,16 +34,19 @@ export function VoiceUser({ tile, large }: VoiceUserProps) { return () => tile.videoTrack?.removeEventListener('ended', onEnded); }, [tile.videoTrack]); - // Attach Video + // Attach Video — use LiveKit's track.attach() to register the element + // with the adaptive stream observer (enables SFU layer switching by viewport size) useEffect(() => { const videoEl = videoRef.current; if (!videoEl) return; - if (activeVideoTrack) { - videoEl.srcObject = new MediaStream([activeVideoTrack]); + const lkTrack = tile.lkVideoTrack; + if (lkTrack) { + lkTrack.attach(videoEl); + return () => { lkTrack.detach(videoEl); }; } else { videoEl.srcObject = null; } - }, [activeVideoTrack]); + }, [tile.lkVideoTrack]); // Context Menu const [volumeMenu, setVolumeMenu] = useState<{ diff --git a/packages/web/src/hooks/useLiveKit.ts b/packages/web/src/hooks/useLiveKit.ts index 6cc57014..c442a321 100644 --- a/packages/web/src/hooks/useLiveKit.ts +++ b/packages/web/src/hooks/useLiveKit.ts @@ -46,6 +46,8 @@ export interface ParticipantInfo { videoTrack: MediaStreamTrack | null; screenTrack: MediaStreamTrack | null; screenAudioTrack: MediaStreamTrack | null; + lkVideoTrack: Track | null; // LiveKit Track for attach/detach (adaptive stream) + lkScreenTrack: Track | null; // LiveKit Track for attach/detach (adaptive stream) } export interface UserTile { @@ -54,6 +56,7 @@ export interface UserTile { participant: ParticipantInfo; videoTrack: MediaStreamTrack | null; // camera only audioTrack: MediaStreamTrack | null; // mic + lkVideoTrack: Track | null; // LiveKit Track for attach/detach } export interface StreamTile { @@ -62,6 +65,7 @@ export interface StreamTile { participant: ParticipantInfo; screenTrack: MediaStreamTrack | null; screenAudioTrack: MediaStreamTrack | null; + lkScreenTrack: Track | null; // LiveKit Track for attach/detach } export type GridTile = UserTile | StreamTile; @@ -69,12 +73,14 @@ export type GridTile = UserTile | StreamTile; export function deriveGridTiles(participants: ParticipantInfo[]): GridTile[] { const tiles: GridTile[] = []; for (const p of participants) { + const hasLiveVideo = p.isCameraOn && p.videoTrack?.readyState === 'live'; tiles.push({ kind: 'user', key: p.identity, participant: p, - videoTrack: (p.isCameraOn && p.videoTrack?.readyState === 'live') ? p.videoTrack : null, + videoTrack: hasLiveVideo ? p.videoTrack : null, audioTrack: p.audioTrack, + lkVideoTrack: hasLiveVideo ? p.lkVideoTrack : null, }); if (p.isScreenSharing) { tiles.push({ @@ -83,6 +89,7 @@ export function deriveGridTiles(participants: ParticipantInfo[]): GridTile[] { participant: p, screenTrack: p.screenTrack, screenAudioTrack: p.screenAudioTrack, + lkScreenTrack: p.lkScreenTrack, }); } } @@ -149,6 +156,8 @@ export function useLiveKit() { let videoTrack: MediaStreamTrack | null = null; let screenTrack: MediaStreamTrack | null = null; let screenAudioTrack: MediaStreamTrack | null = null; + let lkVideoTrack: Track | null = null; + let lkScreenTrack: Track | null = null; let hasScreenSharePublication = false; p.trackPublications.forEach((pub) => { // Detect screen share publication even if unsubscribed @@ -164,8 +173,8 @@ export function useLiveKit() { if (!mt || mt.readyState !== 'live') return; if (pub.source === Track.Source.Microphone) audioTrack = mt; - else if (pub.source === Track.Source.Camera && p.isCameraEnabled) videoTrack = mt; - else if (pub.source === Track.Source.ScreenShare) screenTrack = mt; + else if (pub.source === Track.Source.Camera && p.isCameraEnabled) { videoTrack = mt; lkVideoTrack = track; } + else if (pub.source === Track.Source.ScreenShare) { screenTrack = mt; lkScreenTrack = track; } else if (pub.source === Track.Source.ScreenShareAudio) screenAudioTrack = mt; }); @@ -194,6 +203,8 @@ export function useLiveKit() { videoTrack, screenTrack, screenAudioTrack, + lkVideoTrack, + lkScreenTrack, }); }; processParticipant(r.localParticipant, true); diff --git a/packages/web/src/utils/screenShare.ts b/packages/web/src/utils/screenShare.ts index 4c7f19a4..bb8851bc 100644 --- a/packages/web/src/utils/screenShare.ts +++ b/packages/web/src/utils/screenShare.ts @@ -18,7 +18,7 @@ export interface OverdriveOptions { export interface ScreenShareBuildResult { capture: { width: number; height: number; frameRate: number }; - publish: { videoCodec: 'h264'; videoEncoding: { maxBitrate: number; maxFramerate: number }; simulcast: false }; + publish: { videoCodec: 'vp9'; videoEncoding: { maxBitrate: number; maxFramerate: number }; simulcast: false }; overdrive: OverdriveOptions; contentHint: 'motion' | 'detail'; } @@ -63,7 +63,7 @@ export function buildScreenShareOptions(config: ScreenShareConfig): ScreenShareB return { capture: { width, height, frameRate: fps }, publish: { - videoCodec: 'h264', + videoCodec: 'vp9', videoEncoding: { maxBitrate, maxFramerate: fps }, simulcast: false, }, @@ -99,14 +99,19 @@ export async function applyOverdrive( if (!sender) return; const params = sender.getParameters(); - if (!params.encodings?.[0]) return; + if (!params.encodings?.length) return; - params.encodings[0].maxBitrate = options.maxBitrate; - params.encodings[0].maxFramerate = options.maxFramerate; - params.encodings[0].networkPriority = 'high'; + // Target the highest-quality layer. With simulcast, encodings[0] is the + // lowest layer; our overdrive must hit the top layer so the custom bitrate + // slider controls the full-resolution stream, not the quarter-res one. + // For non-simulcast tracks (single encoding), length - 1 === 0. + const idx = params.encodings.length - 1; + params.encodings[idx]!.maxBitrate = options.maxBitrate; + params.encodings[idx]!.maxFramerate = options.maxFramerate; + params.encodings[idx]!.networkPriority = 'high'; (params as any).degradationPreference = options.degradationPreference; if (options.minBitrate > 0) { - (params.encodings[0] as any).minBitrate = options.minBitrate; + (params.encodings[idx] as any).minBitrate = options.minBitrate; } await sender.setParameters(params); @@ -134,9 +139,9 @@ export async function startScreenShare(room: Room): Promise { // @ts-ignore — LiveKit accepts frameRate at capture level frameRate: opts.capture.frameRate, }, { - videoCodec: 'h264', + videoCodec: opts.publish.videoCodec, videoEncoding: opts.publish.videoEncoding, - simulcast: false, + simulcast: opts.publish.simulcast, } as any); if (!track) {