feat: add screen share audio support
Pass audio: true to setScreenShareEnabled so the browser offers the "Share audio" checkbox. Track ScreenShareAudio from remote participants and play it through a dedicated audio element with the same volume pipeline (including boost >100%).
This commit is contained in:
@@ -89,12 +89,19 @@ export function DmCallView() {
|
|||||||
toggleCamera();
|
toggleCamera();
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleScreenShare = () => {
|
const handleScreenShare = async () => {
|
||||||
const room = getActiveRoom();
|
const room = getActiveRoom();
|
||||||
if (room) {
|
if (!room) return;
|
||||||
room.localParticipant.setScreenShareEnabled(!isScreenSharing);
|
try {
|
||||||
|
if (!isScreenSharing) {
|
||||||
|
await room.localParticipant.setScreenShareEnabled(true, { audio: true });
|
||||||
|
} else {
|
||||||
|
await room.localParticipant.setScreenShareEnabled(false);
|
||||||
|
}
|
||||||
|
toggleScreenShare();
|
||||||
|
} catch (err) {
|
||||||
|
console.error('[DmCallView] Failed to toggle screen share:', err);
|
||||||
}
|
}
|
||||||
toggleScreenShare();
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleEndCall = () => {
|
const handleEndCall = () => {
|
||||||
|
|||||||
@@ -96,7 +96,11 @@ export function VoiceControlBar() {
|
|||||||
const room = getActiveRoom();
|
const room = getActiveRoom();
|
||||||
if (!room) return;
|
if (!room) return;
|
||||||
try {
|
try {
|
||||||
await room.localParticipant.setScreenShareEnabled(!isScreenSharing);
|
if (!isScreenSharing) {
|
||||||
|
await room.localParticipant.setScreenShareEnabled(true, { audio: true });
|
||||||
|
} else {
|
||||||
|
await room.localParticipant.setScreenShareEnabled(false);
|
||||||
|
}
|
||||||
toggleScreenShare();
|
toggleScreenShare();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('[VoiceControlBar] Failed to toggle screen share:', err);
|
console.error('[VoiceControlBar] Failed to toggle screen share:', err);
|
||||||
|
|||||||
@@ -42,7 +42,11 @@ export function VoiceControls() {
|
|||||||
const room = getActiveRoom();
|
const room = getActiveRoom();
|
||||||
if (!room) return;
|
if (!room) return;
|
||||||
try {
|
try {
|
||||||
await room.localParticipant.setScreenShareEnabled(!isScreenSharing);
|
if (!isScreenSharing) {
|
||||||
|
await room.localParticipant.setScreenShareEnabled(true, { audio: true });
|
||||||
|
} else {
|
||||||
|
await room.localParticipant.setScreenShareEnabled(false);
|
||||||
|
}
|
||||||
toggleScreenShare();
|
toggleScreenShare();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('[VoiceControls] Failed to toggle screen share:', err);
|
console.error('[VoiceControls] Failed to toggle screen share:', err);
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ interface VoiceUserProps {
|
|||||||
export function VoiceUser({ participant, large }: VoiceUserProps) {
|
export function VoiceUser({ participant, large }: VoiceUserProps) {
|
||||||
const videoRef = useRef<HTMLVideoElement>(null);
|
const videoRef = useRef<HTMLVideoElement>(null);
|
||||||
const audioRef = useRef<HTMLAudioElement>(null);
|
const audioRef = useRef<HTMLAudioElement>(null);
|
||||||
|
const screenAudioRef = useRef<HTMLAudioElement>(null);
|
||||||
|
|
||||||
const isDeafened = useVoiceStore((s) => s.isDeafened);
|
const isDeafened = useVoiceStore((s) => s.isDeafened);
|
||||||
const outputVolume = useVoiceStore((s) => s.outputVolume);
|
const outputVolume = useVoiceStore((s) => s.outputVolume);
|
||||||
@@ -128,6 +129,80 @@ export function VoiceUser({ participant, large }: VoiceUserProps) {
|
|||||||
}, [outputVolume, perUserVolume, isDeafened, isLocal, participant.audioTrack]);
|
}, [outputVolume, perUserVolume, isDeafened, isLocal, participant.audioTrack]);
|
||||||
|
|
||||||
|
|
||||||
|
// --- SCREEN SHARE AUDIO PIPELINE ---
|
||||||
|
|
||||||
|
const screenBoostGainRef = useRef<GainNode | null>(null);
|
||||||
|
const screenBoostSourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
|
||||||
|
|
||||||
|
// Screen share audio: track attachment
|
||||||
|
useEffect(() => {
|
||||||
|
const audioEl = screenAudioRef.current;
|
||||||
|
if (isLocal || !audioEl || !participant.screenAudioTrack) {
|
||||||
|
if (audioEl) audioEl.srcObject = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const stream = new MediaStream([participant.screenAudioTrack]);
|
||||||
|
if ((audioEl.srcObject as MediaStream)?.id !== stream.id) {
|
||||||
|
audioEl.srcObject = stream;
|
||||||
|
audioEl.play().catch(() => {});
|
||||||
|
}
|
||||||
|
}, [participant.screenAudioTrack, isLocal]);
|
||||||
|
|
||||||
|
// Screen share audio: volume management (mirrors mic audio pipeline)
|
||||||
|
useEffect(() => {
|
||||||
|
const audioEl = screenAudioRef.current;
|
||||||
|
if (isLocal || !audioEl || !participant.screenAudioTrack) return;
|
||||||
|
|
||||||
|
const globalScale = outputVolume / 100;
|
||||||
|
const userScale = perUserVolume / 100;
|
||||||
|
const finalVolume = globalScale * userScale;
|
||||||
|
|
||||||
|
if (isDeafened) {
|
||||||
|
audioEl.muted = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const audioManager = AudioManager.getInstance();
|
||||||
|
const ctx = audioManager.getContext();
|
||||||
|
const isBoosting = finalVolume > 1.0;
|
||||||
|
const isContextReady = ctx && ctx.state === 'running';
|
||||||
|
|
||||||
|
if (isBoosting && isContextReady) {
|
||||||
|
if (!screenBoostGainRef.current && ctx) {
|
||||||
|
const gain = ctx.createGain();
|
||||||
|
const source = ctx.createMediaStreamSource(new MediaStream([participant.screenAudioTrack]));
|
||||||
|
source.connect(gain);
|
||||||
|
gain.connect(ctx.destination);
|
||||||
|
screenBoostGainRef.current = gain;
|
||||||
|
screenBoostSourceRef.current = source;
|
||||||
|
}
|
||||||
|
if (screenBoostGainRef.current && ctx) {
|
||||||
|
screenBoostGainRef.current.gain.setTargetAtTime(finalVolume, ctx.currentTime, 0.01);
|
||||||
|
}
|
||||||
|
audioEl.muted = true;
|
||||||
|
} else {
|
||||||
|
if (screenBoostSourceRef.current) {
|
||||||
|
screenBoostSourceRef.current.disconnect();
|
||||||
|
screenBoostSourceRef.current = null;
|
||||||
|
screenBoostGainRef.current = null;
|
||||||
|
}
|
||||||
|
audioEl.muted = false;
|
||||||
|
audioEl.volume = Math.min(finalVolume, 1.0);
|
||||||
|
if (audioEl.paused) {
|
||||||
|
audioEl.play().catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
if (screenBoostSourceRef.current) {
|
||||||
|
screenBoostSourceRef.current.disconnect();
|
||||||
|
screenBoostSourceRef.current = null;
|
||||||
|
screenBoostGainRef.current = null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}, [outputVolume, perUserVolume, isDeafened, isLocal, participant.screenAudioTrack]);
|
||||||
|
|
||||||
// --- VIDEO & UI ---
|
// --- VIDEO & UI ---
|
||||||
|
|
||||||
const liveScreen = participant.isScreenSharing && participant.screenTrack?.readyState === 'live' ? participant.screenTrack : null;
|
const liveScreen = participant.isScreenSharing && participant.screenTrack?.readyState === 'live' ? participant.screenTrack : null;
|
||||||
@@ -188,6 +263,7 @@ export function VoiceUser({ participant, large }: VoiceUserProps) {
|
|||||||
- PlaysInline is critical for mobile
|
- PlaysInline is critical for mobile
|
||||||
*/}
|
*/}
|
||||||
{!isLocal && <audio ref={audioRef} autoPlay playsInline />}
|
{!isLocal && <audio ref={audioRef} autoPlay playsInline />}
|
||||||
|
{!isLocal && <audio ref={screenAudioRef} autoPlay playsInline />}
|
||||||
|
|
||||||
{hasVideo ? (
|
{hasVideo ? (
|
||||||
<video
|
<video
|
||||||
|
|||||||
@@ -49,6 +49,7 @@ export interface ParticipantInfo {
|
|||||||
audioTrack: MediaStreamTrack | null;
|
audioTrack: MediaStreamTrack | null;
|
||||||
videoTrack: MediaStreamTrack | null;
|
videoTrack: MediaStreamTrack | null;
|
||||||
screenTrack: MediaStreamTrack | null;
|
screenTrack: MediaStreamTrack | null;
|
||||||
|
screenAudioTrack: MediaStreamTrack | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseIdentity(identity: string): { userId: string; username: string } {
|
function parseIdentity(identity: string): { userId: string; username: string } {
|
||||||
@@ -114,6 +115,7 @@ export function useLiveKit() {
|
|||||||
let audioTrack: MediaStreamTrack | null = null;
|
let audioTrack: MediaStreamTrack | null = null;
|
||||||
let videoTrack: MediaStreamTrack | null = null;
|
let videoTrack: MediaStreamTrack | null = null;
|
||||||
let screenTrack: MediaStreamTrack | null = null;
|
let screenTrack: MediaStreamTrack | null = null;
|
||||||
|
let screenAudioTrack: MediaStreamTrack | null = null;
|
||||||
p.trackPublications.forEach((pub) => {
|
p.trackPublications.forEach((pub) => {
|
||||||
const track = pub.track;
|
const track = pub.track;
|
||||||
if (!track) return;
|
if (!track) return;
|
||||||
@@ -125,7 +127,8 @@ export function useLiveKit() {
|
|||||||
|
|
||||||
if (pub.source === Track.Source.Microphone) audioTrack = mt;
|
if (pub.source === Track.Source.Microphone) audioTrack = mt;
|
||||||
else if (pub.source === Track.Source.Camera && p.isCameraEnabled) videoTrack = mt;
|
else if (pub.source === Track.Source.Camera && p.isCameraEnabled) videoTrack = mt;
|
||||||
else if (pub.source === Track.Source.ScreenShare) screenTrack = mt; // Removed isScreenShareEnabled check to rely on track presence
|
else if (pub.source === Track.Source.ScreenShare) screenTrack = mt;
|
||||||
|
else if (pub.source === Track.Source.ScreenShareAudio) screenAudioTrack = mt;
|
||||||
});
|
});
|
||||||
|
|
||||||
const userState = useVoiceStore.getState().voiceUserStates.get(userId);
|
const userState = useVoiceStore.getState().voiceUserStates.get(userId);
|
||||||
@@ -152,7 +155,8 @@ export function useLiveKit() {
|
|||||||
isLocal,
|
isLocal,
|
||||||
audioTrack,
|
audioTrack,
|
||||||
videoTrack,
|
videoTrack,
|
||||||
screenTrack
|
screenTrack,
|
||||||
|
screenAudioTrack,
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
processParticipant(r.localParticipant, true);
|
processParticipant(r.localParticipant, true);
|
||||||
@@ -455,6 +459,7 @@ export function useLiveKit() {
|
|||||||
if (!isScreenSharing) {
|
if (!isScreenSharing) {
|
||||||
const preset = QUALITY_MAP[videoQuality] || AUTO_PRESET;
|
const preset = QUALITY_MAP[videoQuality] || AUTO_PRESET;
|
||||||
const track = await roomRef.current.localParticipant.setScreenShareEnabled(true, {
|
const track = await roomRef.current.localParticipant.setScreenShareEnabled(true, {
|
||||||
|
audio: true,
|
||||||
resolution: VideoPresets.h360.resolution,
|
resolution: VideoPresets.h360.resolution,
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
frameRate: 30,
|
frameRate: 30,
|
||||||
|
|||||||
Reference in New Issue
Block a user