feat: add screen share audio support

Pass audio: true to setScreenShareEnabled so the browser offers the
"Share audio" checkbox. Track ScreenShareAudio from remote participants
and play it through a dedicated audio element with the same volume
pipeline (including boost >100%).
This commit is contained in:
Jannis Braun
2026-02-20 03:16:34 +01:00
parent a8be2dd5da
commit a8656e6a3b
5 changed files with 116 additions and 20 deletions
@@ -89,12 +89,19 @@ export function DmCallView() {
toggleCamera(); toggleCamera();
}; };
const handleScreenShare = () => { const handleScreenShare = async () => {
const room = getActiveRoom(); const room = getActiveRoom();
if (room) { if (!room) return;
room.localParticipant.setScreenShareEnabled(!isScreenSharing); try {
if (!isScreenSharing) {
await room.localParticipant.setScreenShareEnabled(true, { audio: true });
} else {
await room.localParticipant.setScreenShareEnabled(false);
}
toggleScreenShare();
} catch (err) {
console.error('[DmCallView] Failed to toggle screen share:', err);
} }
toggleScreenShare();
}; };
const handleEndCall = () => { const handleEndCall = () => {
@@ -96,7 +96,11 @@ export function VoiceControlBar() {
const room = getActiveRoom(); const room = getActiveRoom();
if (!room) return; if (!room) return;
try { try {
await room.localParticipant.setScreenShareEnabled(!isScreenSharing); if (!isScreenSharing) {
await room.localParticipant.setScreenShareEnabled(true, { audio: true });
} else {
await room.localParticipant.setScreenShareEnabled(false);
}
toggleScreenShare(); toggleScreenShare();
} catch (err) { } catch (err) {
console.error('[VoiceControlBar] Failed to toggle screen share:', err); console.error('[VoiceControlBar] Failed to toggle screen share:', err);
@@ -42,7 +42,11 @@ export function VoiceControls() {
const room = getActiveRoom(); const room = getActiveRoom();
if (!room) return; if (!room) return;
try { try {
await room.localParticipant.setScreenShareEnabled(!isScreenSharing); if (!isScreenSharing) {
await room.localParticipant.setScreenShareEnabled(true, { audio: true });
} else {
await room.localParticipant.setScreenShareEnabled(false);
}
toggleScreenShare(); toggleScreenShare();
} catch (err) { } catch (err) {
console.error('[VoiceControls] Failed to toggle screen share:', err); console.error('[VoiceControls] Failed to toggle screen share:', err);
@@ -12,6 +12,7 @@ interface VoiceUserProps {
export function VoiceUser({ participant, large }: VoiceUserProps) { export function VoiceUser({ participant, large }: VoiceUserProps) {
const videoRef = useRef<HTMLVideoElement>(null); const videoRef = useRef<HTMLVideoElement>(null);
const audioRef = useRef<HTMLAudioElement>(null); const audioRef = useRef<HTMLAudioElement>(null);
const screenAudioRef = useRef<HTMLAudioElement>(null);
const isDeafened = useVoiceStore((s) => s.isDeafened); const isDeafened = useVoiceStore((s) => s.isDeafened);
const outputVolume = useVoiceStore((s) => s.outputVolume); const outputVolume = useVoiceStore((s) => s.outputVolume);
@@ -128,6 +129,80 @@ export function VoiceUser({ participant, large }: VoiceUserProps) {
}, [outputVolume, perUserVolume, isDeafened, isLocal, participant.audioTrack]); }, [outputVolume, perUserVolume, isDeafened, isLocal, participant.audioTrack]);
// --- SCREEN SHARE AUDIO PIPELINE ---
const screenBoostGainRef = useRef<GainNode | null>(null);
const screenBoostSourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
// Screen share audio: track attachment
useEffect(() => {
const audioEl = screenAudioRef.current;
if (isLocal || !audioEl || !participant.screenAudioTrack) {
if (audioEl) audioEl.srcObject = null;
return;
}
const stream = new MediaStream([participant.screenAudioTrack]);
if ((audioEl.srcObject as MediaStream)?.id !== stream.id) {
audioEl.srcObject = stream;
audioEl.play().catch(() => {});
}
}, [participant.screenAudioTrack, isLocal]);
// Screen share audio: volume management (mirrors mic audio pipeline)
useEffect(() => {
const audioEl = screenAudioRef.current;
if (isLocal || !audioEl || !participant.screenAudioTrack) return;
const globalScale = outputVolume / 100;
const userScale = perUserVolume / 100;
const finalVolume = globalScale * userScale;
if (isDeafened) {
audioEl.muted = true;
return;
}
const audioManager = AudioManager.getInstance();
const ctx = audioManager.getContext();
const isBoosting = finalVolume > 1.0;
const isContextReady = ctx && ctx.state === 'running';
if (isBoosting && isContextReady) {
if (!screenBoostGainRef.current && ctx) {
const gain = ctx.createGain();
const source = ctx.createMediaStreamSource(new MediaStream([participant.screenAudioTrack]));
source.connect(gain);
gain.connect(ctx.destination);
screenBoostGainRef.current = gain;
screenBoostSourceRef.current = source;
}
if (screenBoostGainRef.current && ctx) {
screenBoostGainRef.current.gain.setTargetAtTime(finalVolume, ctx.currentTime, 0.01);
}
audioEl.muted = true;
} else {
if (screenBoostSourceRef.current) {
screenBoostSourceRef.current.disconnect();
screenBoostSourceRef.current = null;
screenBoostGainRef.current = null;
}
audioEl.muted = false;
audioEl.volume = Math.min(finalVolume, 1.0);
if (audioEl.paused) {
audioEl.play().catch(() => {});
}
}
return () => {
if (screenBoostSourceRef.current) {
screenBoostSourceRef.current.disconnect();
screenBoostSourceRef.current = null;
screenBoostGainRef.current = null;
}
};
}, [outputVolume, perUserVolume, isDeafened, isLocal, participant.screenAudioTrack]);
// --- VIDEO & UI --- // --- VIDEO & UI ---
const liveScreen = participant.isScreenSharing && participant.screenTrack?.readyState === 'live' ? participant.screenTrack : null; const liveScreen = participant.isScreenSharing && participant.screenTrack?.readyState === 'live' ? participant.screenTrack : null;
@@ -188,6 +263,7 @@ export function VoiceUser({ participant, large }: VoiceUserProps) {
- PlaysInline is critical for mobile - PlaysInline is critical for mobile
*/} */}
{!isLocal && <audio ref={audioRef} autoPlay playsInline />} {!isLocal && <audio ref={audioRef} autoPlay playsInline />}
{!isLocal && <audio ref={screenAudioRef} autoPlay playsInline />}
{hasVideo ? ( {hasVideo ? (
<video <video
+7 -2
View File
@@ -49,6 +49,7 @@ export interface ParticipantInfo {
audioTrack: MediaStreamTrack | null; audioTrack: MediaStreamTrack | null;
videoTrack: MediaStreamTrack | null; videoTrack: MediaStreamTrack | null;
screenTrack: MediaStreamTrack | null; screenTrack: MediaStreamTrack | null;
screenAudioTrack: MediaStreamTrack | null;
} }
function parseIdentity(identity: string): { userId: string; username: string } { function parseIdentity(identity: string): { userId: string; username: string } {
@@ -114,6 +115,7 @@ export function useLiveKit() {
let audioTrack: MediaStreamTrack | null = null; let audioTrack: MediaStreamTrack | null = null;
let videoTrack: MediaStreamTrack | null = null; let videoTrack: MediaStreamTrack | null = null;
let screenTrack: MediaStreamTrack | null = null; let screenTrack: MediaStreamTrack | null = null;
let screenAudioTrack: MediaStreamTrack | null = null;
p.trackPublications.forEach((pub) => { p.trackPublications.forEach((pub) => {
const track = pub.track; const track = pub.track;
if (!track) return; if (!track) return;
@@ -125,7 +127,8 @@ export function useLiveKit() {
if (pub.source === Track.Source.Microphone) audioTrack = mt; if (pub.source === Track.Source.Microphone) audioTrack = mt;
else if (pub.source === Track.Source.Camera && p.isCameraEnabled) videoTrack = mt; else if (pub.source === Track.Source.Camera && p.isCameraEnabled) videoTrack = mt;
else if (pub.source === Track.Source.ScreenShare) screenTrack = mt; // Removed isScreenShareEnabled check to rely on track presence else if (pub.source === Track.Source.ScreenShare) screenTrack = mt;
else if (pub.source === Track.Source.ScreenShareAudio) screenAudioTrack = mt;
}); });
const userState = useVoiceStore.getState().voiceUserStates.get(userId); const userState = useVoiceStore.getState().voiceUserStates.get(userId);
@@ -152,7 +155,8 @@ export function useLiveKit() {
isLocal, isLocal,
audioTrack, audioTrack,
videoTrack, videoTrack,
screenTrack screenTrack,
screenAudioTrack,
}); });
}; };
processParticipant(r.localParticipant, true); processParticipant(r.localParticipant, true);
@@ -455,6 +459,7 @@ export function useLiveKit() {
if (!isScreenSharing) { if (!isScreenSharing) {
const preset = QUALITY_MAP[videoQuality] || AUTO_PRESET; const preset = QUALITY_MAP[videoQuality] || AUTO_PRESET;
const track = await roomRef.current.localParticipant.setScreenShareEnabled(true, { const track = await roomRef.current.localParticipant.setScreenShareEnabled(true, {
audio: true,
resolution: VideoPresets.h360.resolution, resolution: VideoPresets.h360.resolution,
// @ts-ignore // @ts-ignore
frameRate: 30, frameRate: 30,