import { Room, Track, BackupCodecPolicy } from 'livekit-client'; import { useVoiceStore } from '../stores/voiceStore'; import { nativeScreenAudio } from './nativeScreenAudio'; import { isElectron } from '../platform/platform'; import type { ScreenShareConfig } from '../stores/voiceStore'; import { getStreamingLimits } from '../stores/settingsStore'; import { getPublisherPC, getMediaStreamTrack } from './livekitInternals'; import { broadcastVoiceStatus } from './voice'; import { activate as activateHwOverdrive, deactivate as deactivateHwOverdrive } from './hwOverdrive'; import { useUIStore } from '../stores/uiStore'; import { STANDARD_RESOLUTIONS, STANDARD_FRAMERATES, WIDTH_MAP, BITRATE_MATRIX_KBPS, type StandardResolution, } from '@backspace/shared/src/constants'; // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- export interface OverdriveOptions { maxBitrate: number; maxFramerate: number; minBitrate: number; degradationPreference: RTCDegradationPreference; } export interface ScreenShareBuildResult { capture: { width: number; height: number; frameRate: number }; publish: { videoCodec: 'vp9' | 'h264'; videoEncoding: { maxBitrate: number; maxFramerate: number }; simulcast: false; backupCodec?: { codec: 'vp8' | 'h264'; encoding: { maxBitrate: number; maxFramerate: number } }; backupCodecPolicy?: BackupCodecPolicy; }; overdrive: OverdriveOptions; contentHint: 'motion' | 'detail'; } // --------------------------------------------------------------------------- // Camera preset (fixed 720p30 H264, decoupled from screen share) // --------------------------------------------------------------------------- export const CAMERA_PRESET = { resolution: { width: 1280, height: 720 }, encoding: { maxBitrate: 2_000_000, maxFramerate: 30 }, codec: 'h264' as const, } as const; export const CAMERA_OVERDRIVE: OverdriveOptions = { maxBitrate: 2_000_000, maxFramerate: 30, minBitrate: 0, degradationPreference: 'maintain-framerate', }; // --------------------------------------------------------------------------- // Screen share builder — three independent axes → computed result // --------------------------------------------------------------------------- // --------------------------------------------------------------------------- // Resolve a matrix cell: admin override first, then default (all in kbps) // --------------------------------------------------------------------------- function resolveMatrixKbps(height: number, fps: number, overrides: Record | null | undefined): number { const key = `${height}_${fps}`; if (overrides?.[key] != null) return overrides[key]!; return BITRATE_MATRIX_KBPS[height]?.[fps] ?? BITRATE_MATRIX_KBPS[1080]![60]!; } // --------------------------------------------------------------------------- // Native mode — pixel-count-proportional bitrate computation // --------------------------------------------------------------------------- function computeNativeBitrate( capturedWidth: number, capturedHeight: number, fps: number, overrides: Record | null | undefined, ): number { const capturedPixels = capturedWidth * capturedHeight; // Find nearest known resolution tier by pixel count (handles ultrawides correctly) let nearestHeight: StandardResolution = 1080; let nearestDist = Infinity; for (const h of STANDARD_RESOLUTIONS) { const knownPixels = WIDTH_MAP[h] * h; const dist = Math.abs(capturedPixels - knownPixels); if (dist < nearestDist) { nearestDist = dist; nearestHeight = h; } } // Snap to nearest known framerate let nearestFps = 30; let nearestFpsDist = Infinity; for (const f of STANDARD_FRAMERATES) { const dist = Math.abs(fps - f); if (dist < nearestFpsDist) { nearestFpsDist = dist; nearestFps = f; } } const baseKbps = resolveMatrixKbps(nearestHeight, nearestFps, overrides); const nearestPixels = WIDTH_MAP[nearestHeight] * nearestHeight; // Scale proportionally by pixel count and framerate — result in kbps return Math.round(baseKbps * (capturedPixels / nearestPixels) * (fps / nearestFps)); } export function buildScreenShareOptions(config: ScreenShareConfig): ScreenShareBuildResult { const { height, fps, mode, customBitrateKbps } = config; const isNative = height === 'native'; const limits = getStreamingLimits(); const overrides = limits.bitrateMatrixOverrides; // Capture dimensions: sentinel 0 for native (caller skips resolution constraint) const captureWidth = isNative ? 0 : WIDTH_MAP[height as StandardResolution] ?? 1920; const captureHeight = isNative ? 0 : (height as number); // Resolve bitrate in kbps: custom (if allowed) > override > default > native estimate let rawKbps: number; if (customBitrateKbps != null && limits.allowCustomBitrate) { rawKbps = customBitrateKbps; } else if (isNative) { const nearestFps = STANDARD_FRAMERATES.reduce((a, b) => Math.abs(b - fps) < Math.abs(a - fps) ? b : a ); rawKbps = resolveMatrixKbps(2160, nearestFps, overrides); } else { rawKbps = resolveMatrixKbps(height as number, fps, overrides); } // Clamp to instance limits (all in kbps) const clampedKbps = Math.min(Math.max(rawKbps, limits.minBitrateKbps), limits.maxBitrateKbps); // Convert to bps ONLY at the WebRTC boundary const bps = clampedKbps * 1000; const minBps = Math.round(bps * 0.25); // hwOverdrive forces H.264 with SDP profile override for hardware encoding. // Default is always VP9. Both paths get VP8 SIMULCAST backup (dynacast pauses // the backup when no subscriber needs it — near-zero cost). const hwOverdrive = useVoiceStore.getState().hwOverdrive; // Backup encoding: cap at 30fps and proportional bitrate to keep CPU overhead low const backupFps = Math.min(fps, 30); const backupBps = Math.round(bps * (backupFps / fps)); return { capture: { width: captureWidth, height: captureHeight, frameRate: fps }, publish: { videoCodec: hwOverdrive ? 'h264' : 'vp9', videoEncoding: { maxBitrate: bps, maxFramerate: fps }, simulcast: false, backupCodec: { codec: 'vp8' as const, encoding: { maxBitrate: backupBps, maxFramerate: backupFps }, }, backupCodecPolicy: BackupCodecPolicy.SIMULCAST, }, overdrive: { maxBitrate: bps, maxFramerate: fps, minBitrate: minBps, degradationPreference: mode === 'text' ? 'maintain-resolution' : 'balanced', }, contentHint: mode === 'text' ? 'detail' : 'motion', }; } // --------------------------------------------------------------------------- // Shared helper: resolve native-mode overdrive from actual track dimensions // Used by both applyScreenShareOverdrive (screenShare.ts) and updateActiveTracks (useLiveKit.ts) // --------------------------------------------------------------------------- export function resolveNativeOverdrive( mediaTrack: MediaStreamTrack | null | undefined, config: ScreenShareConfig, opts: ScreenShareBuildResult, ): void { const limits = getStreamingLimits(); const effectiveCustom = limits.allowCustomBitrate ? config.customBitrateKbps : null; if (config.height !== 'native' || effectiveCustom != null || !mediaTrack) return; const settings = mediaTrack.getSettings(); if (!settings.width || !settings.height) return; const nativeKbps = computeNativeBitrate(settings.width, settings.height, config.fps, limits.bitrateMatrixOverrides); const clampedKbps = Math.min(Math.max(nativeKbps, limits.minBitrateKbps), limits.maxBitrateKbps); // Convert to bps at the mutation point const bps = clampedKbps * 1000; opts.overdrive.maxBitrate = bps; opts.overdrive.minBitrate = Math.round(bps * 0.25); opts.publish.videoEncoding.maxBitrate = bps; } // --------------------------------------------------------------------------- // Overdrive — forces bitrate/resolution/framerate on RTCRtpSender // --------------------------------------------------------------------------- export async function applyOverdrive( room: Room, source: Track.Source, options: OverdriveOptions, ): Promise { try { const pub = room.localParticipant.getTrackPublications().find(p => p.source === source); if (!pub?.track) return; const pc = getPublisherPC(room); if (!pc) return; const pubMediaTrack = getMediaStreamTrack(pub.track); const senders = pc.getSenders(); const sender = senders.find(s => s.track?.id === pubMediaTrack?.id); if (!sender) return; const params = sender.getParameters(); if (!params.encodings?.length) return; // Target the highest-quality layer. With simulcast, encodings[0] is the // lowest layer; our overdrive must hit the top layer so the custom bitrate // slider controls the full-resolution stream, not the quarter-res one. // For non-simulcast tracks (single encoding), length - 1 === 0. const idx = params.encodings.length - 1; params.encodings[idx]!.maxBitrate = options.maxBitrate; params.encodings[idx]!.maxFramerate = options.maxFramerate; params.encodings[idx]!.networkPriority = 'high'; (params as any).degradationPreference = options.degradationPreference; if (options.minBitrate > 0) { (params.encodings[idx] as any).minBitrate = options.minBitrate; } await sender.setParameters(params); } catch (err) { console.warn('[ScreenShare] Failed to apply overdrive:', err); } } // --------------------------------------------------------------------------- // Start screen sharing — single path via setScreenShareEnabled() // In Electron, getDisplayMedia() is intercepted by setDisplayMediaRequestHandler // in the main process, which shows the custom picker automatically. // --------------------------------------------------------------------------- export async function startScreenShare(room: Room): Promise { console.log('[SS] startScreenShare called, room state:', room.state); const config = useVoiceStore.getState().screenShareConfig; const hwOverdrive = useVoiceStore.getState().hwOverdrive; const opts = buildScreenShareOptions(config); // Activate SDP profile override before WebRTC negotiation if (hwOverdrive) { activateHwOverdrive(); } try { // For native mode: omit resolution constraint to capture at display's full native resolution const captureOptions: any = { // In the desktop app audio never comes through this stream: Electron // ignores these constraints and its loopback would carry our own output // back into the share. The native capture below supplies it instead. audio: config.shareAudio && !isElectron() ? { // Chrome 141+: exclude this tab's own audio from system audio capture // @ts-ignore — restrictOwnAudio is not yet in all TS type definitions restrictOwnAudio: true, echoCancellation: false, noiseSuppression: false, autoGainControl: false, channelCount: 2, } : false, frameRate: opts.capture.frameRate, }; if (opts.capture.width > 0 && opts.capture.height > 0) { captureOptions.resolution = { width: opts.capture.width, height: opts.capture.height }; } const track = await room.localParticipant.setScreenShareEnabled(true, captureOptions, { videoCodec: opts.publish.videoCodec, videoEncoding: opts.publish.videoEncoding, // LiveKit uses screenShareEncoding (not videoEncoding) for screen share tracks. // Without this, the default ScreenSharePresets.h1080fps15 caps at 15fps. screenShareEncoding: opts.publish.videoEncoding, simulcast: opts.publish.simulcast, ...(opts.publish.backupCodec ? { backupCodec: opts.publish.backupCodec, backupCodecPolicy: opts.publish.backupCodecPolicy, } : {}), }); console.log('[SS] setScreenShareEnabled returned:', !!track); if (!track) { if (hwOverdrive) deactivateHwOverdrive(); return false; } // Desktop only, and only after the video track exists: publishing the audio // first would briefly show a share with sound and no picture. A failure // here leaves the share running silently rather than tearing it down. if (isElectron() && config.shareAudio) { const ok = await nativeScreenAudio.start(room); if (!ok) console.warn('[SS] native system audio unavailable — sharing without sound'); } // Set content hint from builder (motion for gaming, detail for text) const screenPub = room.localParticipant.getTrackPublications() .find(p => p.source === Track.Source.ScreenShare); if (screenPub?.track?.mediaStreamTrack) { screenPub.track.mediaStreamTrack.contentHint = opts.contentHint; } useVoiceStore.setState({ isScreenSharing: true }); applyScreenShareOverdrive(room); // Schedule hardware encoder detection if (hwOverdrive) { scheduleEncoderDetection(room); } return true; } catch (err) { console.error('[ScreenShare] Failed to start screen share:', err); if (hwOverdrive) deactivateHwOverdrive(); // Loopback unsupported (Linux without pulse, macOS without Catap) makes // the whole getDisplayMedia call reject. No auto-retry: the picker // selection was consumed, retrying would re-prompt it. if (config.shareAudio && err instanceof Error && err.name !== 'NotAllowedError') { useUIStore.getState().addToast( 'Could not start stream with system audio. Disable "Share system audio" in the picker if your system does not support it.', 'warning', 8000, ); } return false; } } // --------------------------------------------------------------------------- // Shared overdrive scheduling // --------------------------------------------------------------------------- function applyScreenShareOverdrive(room: Room): void { // Overdrive at 2s — after WebRTC finishes negotiation setTimeout(async () => { if (!useVoiceStore.getState().isScreenSharing) return; const freshOpts = buildScreenShareOptions(useVoiceStore.getState().screenShareConfig); const screenPub = room.localParticipant.getTrackPublications() .find(p => p.source === Track.Source.ScreenShare); if (screenPub?.track?.mediaStreamTrack) { if (freshOpts.capture.width > 0 && freshOpts.capture.height > 0) { // Standard mode: apply resolution + frameRate together await screenPub.track.mediaStreamTrack.applyConstraints({ width: { ideal: freshOpts.capture.width }, height: { ideal: freshOpts.capture.height }, frameRate: { ideal: freshOpts.capture.frameRate, min: 15 }, }); } else { // Native mode: apply frameRate only — never pass 0 to width/height await screenPub.track.mediaStreamTrack.applyConstraints({ frameRate: { ideal: freshOpts.capture.frameRate, min: 15 }, }); } screenPub.track.mediaStreamTrack.contentHint = freshOpts.contentHint; // For native mode, compute correct bitrate from actual track dimensions resolveNativeOverdrive(screenPub.track.mediaStreamTrack, useVoiceStore.getState().screenShareConfig, freshOpts); } await applyOverdrive(room, Track.Source.ScreenShare, freshOpts.overdrive); }, 2000); // Second overdrive at 5s — safety net for slow BWE convergence setTimeout(async () => { if (!useVoiceStore.getState().isScreenSharing) return; const freshOpts = buildScreenShareOptions(useVoiceStore.getState().screenShareConfig); const screenPub5 = room.localParticipant.getTrackPublications() .find(p => p.source === Track.Source.ScreenShare); if (screenPub5?.track?.mediaStreamTrack) { resolveNativeOverdrive(screenPub5.track.mediaStreamTrack, useVoiceStore.getState().screenShareConfig, freshOpts); } await applyOverdrive(room, Track.Source.ScreenShare, freshOpts.overdrive); }, 5000); } // --------------------------------------------------------------------------- // Hardware encoder detection — checks WebRTC stats after stream starts // --------------------------------------------------------------------------- function scheduleEncoderDetection(room: Room): void { setTimeout(async () => { if (!useVoiceStore.getState().isScreenSharing) return; if (!useVoiceStore.getState().hwOverdrive) return; try { const pc = getPublisherPC(room); if (!pc) return; const screenPub = room.localParticipant.getTrackPublications() .find(p => p.source === Track.Source.ScreenShare); if (!screenPub?.track) return; const mediaTrack = getMediaStreamTrack(screenPub.track); if (!mediaTrack) return; const sender = pc.getSenders().find(s => s.track?.id === mediaTrack.id); if (!sender) return; const stats = await sender.getStats(); let encoderImpl: string | null = null; stats.forEach((report: any) => { if (report.type === 'outbound-rtp' && (report.kind === 'video' || report.mediaType === 'video')) { encoderImpl = report.encoderImplementation ?? null; } }); if (encoderImpl && /openh264/i.test(encoderImpl)) { useUIStore.getState().addToast( 'Hardware encoder not available — using software fallback. Switch to VP9 for better performance.', 'warning', 8000, ); } } catch { // Non-critical — silently ignore detection failures } }, 4000); } // --------------------------------------------------------------------------- // Stop screen sharing // --------------------------------------------------------------------------- export async function stopScreenShare(room: Room): Promise { // Stopped first: leaving the native capture running would keep reading system // audio after the share is gone. await nativeScreenAudio.stop(); try { await room.localParticipant.setScreenShareEnabled(false); } catch (err) { console.error('[ScreenShare] Failed to stop screen share:', err); } deactivateHwOverdrive(); useVoiceStore.setState({ isScreenSharing: false, hwOverdrive: false }); } // --------------------------------------------------------------------------- // Change screen share source — stops current stream, re-triggers picker // --------------------------------------------------------------------------- export async function changeScreenShare(room: Room): Promise { await stopScreenShare(room); setTimeout(async () => { await startScreenShare(room); }, 200); } // --------------------------------------------------------------------------- // OS-level "Stop sharing" handler // --------------------------------------------------------------------------- export function handleScreenShareUnpublished(): void { deactivateHwOverdrive(); useVoiceStore.setState({ isScreenSharing: false, hwOverdrive: false }); broadcastVoiceStatus(); }