feat: add screen share audio support
Pass audio: true to setScreenShareEnabled so the browser offers the "Share audio" checkbox. Track ScreenShareAudio from remote participants and play it through a dedicated audio element with the same volume pipeline (including boost >100%).
This commit is contained in:
@@ -89,12 +89,19 @@ export function DmCallView() {
|
||||
toggleCamera();
|
||||
};
|
||||
|
||||
const handleScreenShare = () => {
|
||||
const handleScreenShare = async () => {
|
||||
const room = getActiveRoom();
|
||||
if (room) {
|
||||
room.localParticipant.setScreenShareEnabled(!isScreenSharing);
|
||||
if (!room) return;
|
||||
try {
|
||||
if (!isScreenSharing) {
|
||||
await room.localParticipant.setScreenShareEnabled(true, { audio: true });
|
||||
} else {
|
||||
await room.localParticipant.setScreenShareEnabled(false);
|
||||
}
|
||||
toggleScreenShare();
|
||||
} catch (err) {
|
||||
console.error('[DmCallView] Failed to toggle screen share:', err);
|
||||
}
|
||||
toggleScreenShare();
|
||||
};
|
||||
|
||||
const handleEndCall = () => {
|
||||
|
||||
@@ -96,7 +96,11 @@ export function VoiceControlBar() {
|
||||
const room = getActiveRoom();
|
||||
if (!room) return;
|
||||
try {
|
||||
await room.localParticipant.setScreenShareEnabled(!isScreenSharing);
|
||||
if (!isScreenSharing) {
|
||||
await room.localParticipant.setScreenShareEnabled(true, { audio: true });
|
||||
} else {
|
||||
await room.localParticipant.setScreenShareEnabled(false);
|
||||
}
|
||||
toggleScreenShare();
|
||||
} catch (err) {
|
||||
console.error('[VoiceControlBar] Failed to toggle screen share:', err);
|
||||
|
||||
@@ -42,7 +42,11 @@ export function VoiceControls() {
|
||||
const room = getActiveRoom();
|
||||
if (!room) return;
|
||||
try {
|
||||
await room.localParticipant.setScreenShareEnabled(!isScreenSharing);
|
||||
if (!isScreenSharing) {
|
||||
await room.localParticipant.setScreenShareEnabled(true, { audio: true });
|
||||
} else {
|
||||
await room.localParticipant.setScreenShareEnabled(false);
|
||||
}
|
||||
toggleScreenShare();
|
||||
} catch (err) {
|
||||
console.error('[VoiceControls] Failed to toggle screen share:', err);
|
||||
|
||||
@@ -12,6 +12,7 @@ interface VoiceUserProps {
|
||||
export function VoiceUser({ participant, large }: VoiceUserProps) {
|
||||
const videoRef = useRef<HTMLVideoElement>(null);
|
||||
const audioRef = useRef<HTMLAudioElement>(null);
|
||||
const screenAudioRef = useRef<HTMLAudioElement>(null);
|
||||
|
||||
const isDeafened = useVoiceStore((s) => s.isDeafened);
|
||||
const outputVolume = useVoiceStore((s) => s.outputVolume);
|
||||
@@ -128,6 +129,80 @@ export function VoiceUser({ participant, large }: VoiceUserProps) {
|
||||
}, [outputVolume, perUserVolume, isDeafened, isLocal, participant.audioTrack]);
|
||||
|
||||
|
||||
// --- SCREEN SHARE AUDIO PIPELINE ---
|
||||
|
||||
const screenBoostGainRef = useRef<GainNode | null>(null);
|
||||
const screenBoostSourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
|
||||
|
||||
// Screen share audio: track attachment
|
||||
useEffect(() => {
|
||||
const audioEl = screenAudioRef.current;
|
||||
if (isLocal || !audioEl || !participant.screenAudioTrack) {
|
||||
if (audioEl) audioEl.srcObject = null;
|
||||
return;
|
||||
}
|
||||
|
||||
const stream = new MediaStream([participant.screenAudioTrack]);
|
||||
if ((audioEl.srcObject as MediaStream)?.id !== stream.id) {
|
||||
audioEl.srcObject = stream;
|
||||
audioEl.play().catch(() => {});
|
||||
}
|
||||
}, [participant.screenAudioTrack, isLocal]);
|
||||
|
||||
// Screen share audio: volume management (mirrors mic audio pipeline)
|
||||
useEffect(() => {
|
||||
const audioEl = screenAudioRef.current;
|
||||
if (isLocal || !audioEl || !participant.screenAudioTrack) return;
|
||||
|
||||
const globalScale = outputVolume / 100;
|
||||
const userScale = perUserVolume / 100;
|
||||
const finalVolume = globalScale * userScale;
|
||||
|
||||
if (isDeafened) {
|
||||
audioEl.muted = true;
|
||||
return;
|
||||
}
|
||||
|
||||
const audioManager = AudioManager.getInstance();
|
||||
const ctx = audioManager.getContext();
|
||||
const isBoosting = finalVolume > 1.0;
|
||||
const isContextReady = ctx && ctx.state === 'running';
|
||||
|
||||
if (isBoosting && isContextReady) {
|
||||
if (!screenBoostGainRef.current && ctx) {
|
||||
const gain = ctx.createGain();
|
||||
const source = ctx.createMediaStreamSource(new MediaStream([participant.screenAudioTrack]));
|
||||
source.connect(gain);
|
||||
gain.connect(ctx.destination);
|
||||
screenBoostGainRef.current = gain;
|
||||
screenBoostSourceRef.current = source;
|
||||
}
|
||||
if (screenBoostGainRef.current && ctx) {
|
||||
screenBoostGainRef.current.gain.setTargetAtTime(finalVolume, ctx.currentTime, 0.01);
|
||||
}
|
||||
audioEl.muted = true;
|
||||
} else {
|
||||
if (screenBoostSourceRef.current) {
|
||||
screenBoostSourceRef.current.disconnect();
|
||||
screenBoostSourceRef.current = null;
|
||||
screenBoostGainRef.current = null;
|
||||
}
|
||||
audioEl.muted = false;
|
||||
audioEl.volume = Math.min(finalVolume, 1.0);
|
||||
if (audioEl.paused) {
|
||||
audioEl.play().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
return () => {
|
||||
if (screenBoostSourceRef.current) {
|
||||
screenBoostSourceRef.current.disconnect();
|
||||
screenBoostSourceRef.current = null;
|
||||
screenBoostGainRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [outputVolume, perUserVolume, isDeafened, isLocal, participant.screenAudioTrack]);
|
||||
|
||||
// --- VIDEO & UI ---
|
||||
|
||||
const liveScreen = participant.isScreenSharing && participant.screenTrack?.readyState === 'live' ? participant.screenTrack : null;
|
||||
@@ -188,6 +263,7 @@ export function VoiceUser({ participant, large }: VoiceUserProps) {
|
||||
- PlaysInline is critical for mobile
|
||||
*/}
|
||||
{!isLocal && <audio ref={audioRef} autoPlay playsInline />}
|
||||
{!isLocal && <audio ref={screenAudioRef} autoPlay playsInline />}
|
||||
|
||||
{hasVideo ? (
|
||||
<video
|
||||
|
||||
@@ -49,6 +49,7 @@ export interface ParticipantInfo {
|
||||
audioTrack: MediaStreamTrack | null;
|
||||
videoTrack: MediaStreamTrack | null;
|
||||
screenTrack: MediaStreamTrack | null;
|
||||
screenAudioTrack: MediaStreamTrack | null;
|
||||
}
|
||||
|
||||
function parseIdentity(identity: string): { userId: string; username: string } {
|
||||
@@ -114,18 +115,20 @@ export function useLiveKit() {
|
||||
let audioTrack: MediaStreamTrack | null = null;
|
||||
let videoTrack: MediaStreamTrack | null = null;
|
||||
let screenTrack: MediaStreamTrack | null = null;
|
||||
let screenAudioTrack: MediaStreamTrack | null = null;
|
||||
p.trackPublications.forEach((pub) => {
|
||||
const track = pub.track;
|
||||
if (!track) return;
|
||||
// Strict check: Track must be subscribed AND not muted to be considered "active"
|
||||
if (pub.isMuted || !pub.isSubscribed) return;
|
||||
|
||||
|
||||
const mt = track.mediaStreamTrack;
|
||||
if (!mt || mt.readyState !== 'live') return;
|
||||
|
||||
|
||||
if (pub.source === Track.Source.Microphone) audioTrack = mt;
|
||||
else if (pub.source === Track.Source.Camera && p.isCameraEnabled) videoTrack = mt;
|
||||
else if (pub.source === Track.Source.ScreenShare) screenTrack = mt; // Removed isScreenShareEnabled check to rely on track presence
|
||||
else if (pub.source === Track.Source.ScreenShare) screenTrack = mt;
|
||||
else if (pub.source === Track.Source.ScreenShareAudio) screenAudioTrack = mt;
|
||||
});
|
||||
|
||||
const userState = useVoiceStore.getState().voiceUserStates.get(userId);
|
||||
@@ -140,19 +143,20 @@ export function useLiveKit() {
|
||||
if (userState) isPartMuted = userState.isMuted;
|
||||
}
|
||||
|
||||
allParticipants.push({
|
||||
identity: p.identity,
|
||||
userId,
|
||||
username,
|
||||
isSpeaking: p.isSpeaking,
|
||||
isMuted: isPartMuted,
|
||||
isDeafened: isPartDeafened,
|
||||
allParticipants.push({
|
||||
identity: p.identity,
|
||||
userId,
|
||||
username,
|
||||
isSpeaking: p.isSpeaking,
|
||||
isMuted: isPartMuted,
|
||||
isDeafened: isPartDeafened,
|
||||
isCameraOn: !!videoTrack, // Strictly derived from active track
|
||||
isScreenSharing: !!screenTrack, // Strictly derived from active track
|
||||
isLocal,
|
||||
audioTrack,
|
||||
videoTrack,
|
||||
screenTrack
|
||||
isLocal,
|
||||
audioTrack,
|
||||
videoTrack,
|
||||
screenTrack,
|
||||
screenAudioTrack,
|
||||
});
|
||||
};
|
||||
processParticipant(r.localParticipant, true);
|
||||
@@ -455,6 +459,7 @@ export function useLiveKit() {
|
||||
if (!isScreenSharing) {
|
||||
const preset = QUALITY_MAP[videoQuality] || AUTO_PRESET;
|
||||
const track = await roomRef.current.localParticipant.setScreenShareEnabled(true, {
|
||||
audio: true,
|
||||
resolution: VideoPresets.h360.resolution,
|
||||
// @ts-ignore
|
||||
frameRate: 30,
|
||||
|
||||
Reference in New Issue
Block a user