From 4bb69851d3ad7c9ebb073a765290bce768867ea3 Mon Sep 17 00:00:00 2001
From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com>
Date: Thu, 26 Feb 2026 02:33:18 +0100
Subject: [PATCH] fix: use LiveKit track.attach() for adaptive stream and
 switch screen share to VP9 single-layer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Camera was stuck at 180p because our custom <video> rendering bypassed
LiveKit's adaptive stream observer. Replaced manual srcObject binding
with track.attach()/detach() in VoiceUser, StreamTile, and PictureInPicture
so the SFU receives viewport dimensions and forwards the correct
H.264 simulcast layer.

Screen share VP9 SVC with L3T3 spatial layers failed because hardware
VP9 encoders (NVENC, QSV, VCE) don't support spatial scalability —
Chrome silently degrades to L1T1. Reverted to VP9 single-layer
(simulcast: false, no scalabilityMode). Also targets encodings[length-1]
in applyOverdrive() for correct simulcast layer targeting.
---
 .../src/components/voice/PictureInPicture.tsx | 17 ++++++++++----
 .../web/src/components/voice/StreamTile.tsx   | 11 +++++----
 .../web/src/components/voice/VoiceUser.tsx    | 11 +++++----
 packages/web/src/hooks/useLiveKit.ts          | 17 +++++++++++---
 packages/web/src/utils/screenShare.ts         | 23 +++++++++++--------
 5 files changed, 55 insertions(+), 24 deletions(-)

diff --git a/packages/web/src/components/voice/PictureInPicture.tsx b/packages/web/src/components/voice/PictureInPicture.tsx
index 1a2e839a..43955f58 100644
--- a/packages/web/src/components/voice/PictureInPicture.tsx
+++ b/packages/web/src/components/voice/PictureInPicture.tsx
@@ -122,17 +122,26 @@ export function PictureInPicture() {
     return 'Call';
   }, [currentVoiceChannelId, channels]);
 
-  // Video track attachment
+  // Derive the LiveKit Track from the selected stream's participant
+  const lkTrack = selectedStream
+    ? (selectedStream.type === 'screen'
+        ? selectedStream.participant.lkScreenTrack
+        : selectedStream.participant.lkVideoTrack)
+    : null;
+
+  // Video track attachment — use LiveKit's track.attach() to register the element
+  // with the adaptive stream observer (enables SFU layer switching by viewport size)
   // shouldShow in deps ensures re-run when PiP becomes visible (videoRef was null before)
   useEffect(() => {
     const videoEl = videoRef.current;
     if (!videoEl) return;
-    if (selectedStream?.track) {
-      videoEl.srcObject = new MediaStream([selectedStream.track]);
+    if (lkTrack) {
+      lkTrack.attach(videoEl);
+      return () => { lkTrack.detach(videoEl); };
     } else {
       videoEl.srcObject = null;
     }
-  }, [selectedStream?.track, shouldShow]);
+  }, [lkTrack, shouldShow]);
 
   // Initialize position to bottom-right
   useEffect(() => {
diff --git a/packages/web/src/components/voice/StreamTile.tsx b/packages/web/src/components/voice/StreamTile.tsx
index c679a500..b4236887 100644
--- a/packages/web/src/components/voice/StreamTile.tsx
+++ b/packages/web/src/components/voice/StreamTile.tsx
@@ -29,6 +29,7 @@ export function StreamTile({ tile, large }: StreamTileProps) {
   const isStreamMuted = streamMutes.get(userId) ?? false;
 
   const liveScreenTrack = tile.screenTrack?.readyState === 'live' ? tile.screenTrack : null;
+  const liveLkScreenTrack = liveScreenTrack ? tile.lkScreenTrack : null;
 
   // Quality badge state
   const [qualityBadge, setQualityBadge] = useState<string>('');
@@ -37,16 +38,18 @@ export function StreamTile({ tile, large }: StreamTileProps) {
   const [contextMenu, setContextMenu] = useState<{ x: number; y: number } | null>(null);
   const [qualityPopoverOpen, setQualityPopoverOpen] = useState(false);
 
-  // --- VIDEO ---
+  // --- VIDEO --- use LiveKit's track.attach() to register the element
+  // with the adaptive stream observer (enables SFU layer switching by viewport size)
   useEffect(() => {
     const videoEl = videoRef.current;
     if (!videoEl) return;
-    if (liveScreenTrack) {
-      videoEl.srcObject = new MediaStream([liveScreenTrack]);
+    if (liveLkScreenTrack) {
+      liveLkScreenTrack.attach(videoEl);
+      return () => { liveLkScreenTrack.detach(videoEl); };
     } else {
       videoEl.srcObject = null;
     }
-  }, [liveScreenTrack]);
+  }, [liveLkScreenTrack]);
 
   // Quality badge (poll every 3s)
   useEffect(() => {
diff --git a/packages/web/src/components/voice/VoiceUser.tsx b/packages/web/src/components/voice/VoiceUser.tsx
index cd022ecc..85a25f34 100644
--- a/packages/web/src/components/voice/VoiceUser.tsx
+++ b/packages/web/src/components/voice/VoiceUser.tsx
@@ -34,16 +34,19 @@ export function VoiceUser({ tile, large }: VoiceUserProps) {
     return () => tile.videoTrack?.removeEventListener('ended', onEnded);
   }, [tile.videoTrack]);
 
-  // Attach Video
+  // Attach Video — use LiveKit's track.attach() to register the element
+  // with the adaptive stream observer (enables SFU layer switching by viewport size)
   useEffect(() => {
     const videoEl = videoRef.current;
     if (!videoEl) return;
-    if (activeVideoTrack) {
-      videoEl.srcObject = new MediaStream([activeVideoTrack]);
+    const lkTrack = tile.lkVideoTrack;
+    if (lkTrack) {
+      lkTrack.attach(videoEl);
+      return () => { lkTrack.detach(videoEl); };
     } else {
       videoEl.srcObject = null;
     }
-  }, [activeVideoTrack]);
+  }, [tile.lkVideoTrack]);
 
   // Context Menu
   const [volumeMenu, setVolumeMenu] = useState<{
diff --git a/packages/web/src/hooks/useLiveKit.ts b/packages/web/src/hooks/useLiveKit.ts
index 6cc57014..c442a321 100644
--- a/packages/web/src/hooks/useLiveKit.ts
+++ b/packages/web/src/hooks/useLiveKit.ts
@@ -46,6 +46,8 @@ export interface ParticipantInfo {
   videoTrack: MediaStreamTrack | null;
   screenTrack: MediaStreamTrack | null;
   screenAudioTrack: MediaStreamTrack | null;
+  lkVideoTrack: Track | null;   // LiveKit Track for attach/detach (adaptive stream)
+  lkScreenTrack: Track | null;  // LiveKit Track for attach/detach (adaptive stream)
 }
 
 export interface UserTile {
@@ -54,6 +56,7 @@ export interface UserTile {
   participant: ParticipantInfo;
   videoTrack: MediaStreamTrack | null; // camera only
   audioTrack: MediaStreamTrack | null; // mic
+  lkVideoTrack: Track | null;         // LiveKit Track for attach/detach
 }
 
 export interface StreamTile {
@@ -62,6 +65,7 @@ export interface StreamTile {
   participant: ParticipantInfo;
   screenTrack: MediaStreamTrack | null;
   screenAudioTrack: MediaStreamTrack | null;
+  lkScreenTrack: Track | null;        // LiveKit Track for attach/detach
 }
 
 export type GridTile = UserTile | StreamTile;
@@ -69,12 +73,14 @@ export type GridTile = UserTile | StreamTile;
 export function deriveGridTiles(participants: ParticipantInfo[]): GridTile[] {
   const tiles: GridTile[] = [];
   for (const p of participants) {
+    const hasLiveVideo = p.isCameraOn && p.videoTrack?.readyState === 'live';
     tiles.push({
       kind: 'user',
       key: p.identity,
       participant: p,
-      videoTrack: (p.isCameraOn && p.videoTrack?.readyState === 'live') ? p.videoTrack : null,
+      videoTrack: hasLiveVideo ? p.videoTrack : null,
       audioTrack: p.audioTrack,
+      lkVideoTrack: hasLiveVideo ? p.lkVideoTrack : null,
     });
     if (p.isScreenSharing) {
       tiles.push({
@@ -83,6 +89,7 @@ export function deriveGridTiles(participants: ParticipantInfo[]): GridTile[] {
         participant: p,
         screenTrack: p.screenTrack,
         screenAudioTrack: p.screenAudioTrack,
+        lkScreenTrack: p.lkScreenTrack,
       });
     }
   }
@@ -149,6 +156,8 @@ export function useLiveKit() {
       let videoTrack: MediaStreamTrack | null = null;
       let screenTrack: MediaStreamTrack | null = null;
       let screenAudioTrack: MediaStreamTrack | null = null;
+      let lkVideoTrack: Track | null = null;
+      let lkScreenTrack: Track | null = null;
       let hasScreenSharePublication = false;
       p.trackPublications.forEach((pub) => {
         // Detect screen share publication even if unsubscribed
@@ -164,8 +173,8 @@ export function useLiveKit() {
         if (!mt || mt.readyState !== 'live') return;
 
         if (pub.source === Track.Source.Microphone) audioTrack = mt;
-        else if (pub.source === Track.Source.Camera && p.isCameraEnabled) videoTrack = mt;
-        else if (pub.source === Track.Source.ScreenShare) screenTrack = mt;
+        else if (pub.source === Track.Source.Camera && p.isCameraEnabled) { videoTrack = mt; lkVideoTrack = track; }
+        else if (pub.source === Track.Source.ScreenShare) { screenTrack = mt; lkScreenTrack = track; }
         else if (pub.source === Track.Source.ScreenShareAudio) screenAudioTrack = mt;
       });
 
@@ -194,6 +203,8 @@ export function useLiveKit() {
         videoTrack,
         screenTrack,
         screenAudioTrack,
+        lkVideoTrack,
+        lkScreenTrack,
       });
     };
     processParticipant(r.localParticipant, true);
diff --git a/packages/web/src/utils/screenShare.ts b/packages/web/src/utils/screenShare.ts
index 4c7f19a4..bb8851bc 100644
--- a/packages/web/src/utils/screenShare.ts
+++ b/packages/web/src/utils/screenShare.ts
@@ -18,7 +18,7 @@ export interface OverdriveOptions {
 
 export interface ScreenShareBuildResult {
   capture: { width: number; height: number; frameRate: number };
-  publish: { videoCodec: 'h264'; videoEncoding: { maxBitrate: number; maxFramerate: number }; simulcast: false };
+  publish: { videoCodec: 'vp9'; videoEncoding: { maxBitrate: number; maxFramerate: number }; simulcast: false };
   overdrive: OverdriveOptions;
   contentHint: 'motion' | 'detail';
 }
@@ -63,7 +63,7 @@ export function buildScreenShareOptions(config: ScreenShareConfig): ScreenShareB
   return {
     capture: { width, height, frameRate: fps },
     publish: {
-      videoCodec: 'h264',
+      videoCodec: 'vp9',
       videoEncoding: { maxBitrate, maxFramerate: fps },
       simulcast: false,
     },
@@ -99,14 +99,19 @@ export async function applyOverdrive(
     if (!sender) return;
 
     const params = sender.getParameters();
-    if (!params.encodings?.[0]) return;
+    if (!params.encodings?.length) return;
 
-    params.encodings[0].maxBitrate = options.maxBitrate;
-    params.encodings[0].maxFramerate = options.maxFramerate;
-    params.encodings[0].networkPriority = 'high';
+    // Target the highest-quality layer. With simulcast, encodings[0] is the
+    // lowest layer; our overdrive must hit the top layer so the custom bitrate
+    // slider controls the full-resolution stream, not the quarter-res one.
+    // For non-simulcast tracks (single encoding), length - 1 === 0.
+    const idx = params.encodings.length - 1;
+    params.encodings[idx]!.maxBitrate = options.maxBitrate;
+    params.encodings[idx]!.maxFramerate = options.maxFramerate;
+    params.encodings[idx]!.networkPriority = 'high';
     (params as any).degradationPreference = options.degradationPreference;
     if (options.minBitrate > 0) {
-      (params.encodings[0] as any).minBitrate = options.minBitrate;
+      (params.encodings[idx] as any).minBitrate = options.minBitrate;
     }
 
     await sender.setParameters(params);
@@ -134,9 +139,9 @@ export async function startScreenShare(room: Room): Promise<boolean> {
       // @ts-ignore — LiveKit accepts frameRate at capture level
       frameRate: opts.capture.frameRate,
     }, {
-      videoCodec: 'h264',
+      videoCodec: opts.publish.videoCodec,
       videoEncoding: opts.publish.videoEncoding,
-      simulcast: false,
+      simulcast: opts.publish.simulcast,
     } as any);
 
     if (!track) {