From 95217f0343881ede2d86874c30c6fa206ebef05b Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Thu, 19 Feb 2026 22:13:48 +0100 Subject: [PATCH] Overhaul audio architecture for robust cross-browser reliability --- packages/web/src/audio/AudioManager.ts | 144 +++++++++++ .../web/src/components/layout/AppLayout.tsx | 28 +++ .../src/components/layout/ChannelSidebar.tsx | 100 +++----- .../src/components/voice/VoiceControlBar.tsx | 16 -- .../web/src/components/voice/VoiceUser.tsx | 234 +++++++++--------- packages/web/src/hooks/useLiveKit.ts | 187 +++++++------- packages/web/src/stores/voiceStore.ts | 23 +- 7 files changed, 439 insertions(+), 293 deletions(-) create mode 100644 packages/web/src/audio/AudioManager.ts diff --git a/packages/web/src/audio/AudioManager.ts b/packages/web/src/audio/AudioManager.ts new file mode 100644 index 00000000..53030886 --- /dev/null +++ b/packages/web/src/audio/AudioManager.ts @@ -0,0 +1,144 @@ +export class AudioManager { + private static instance: AudioManager | null = null; + private ctx: AudioContext | null = null; + private inputGain: GainNode | null = null; + private inputSource: MediaStreamAudioSourceNode | null = null; + private inputDestination: MediaStreamAudioDestinationNode | null = null; + private silentGain: GainNode | null = null; + private analyser: AnalyserNode | null = null; + + private currentInputDeviceId: string = 'default'; + private currentStream: MediaStream | null = null; + private isInitialized = false; + + private listeners: Set<() => void> = new Set(); + + private constructor() {} + + static getInstance(): AudioManager { + if (!AudioManager.instance) { + AudioManager.instance = new AudioManager(); + } + return AudioManager.instance; + } + + private initContext() { + if (this.ctx) return; + const AudioContextClass = window.AudioContext || (window as any).webkitAudioContext; + this.ctx = new AudioContextClass(); + + this.inputGain = this.ctx.createGain(); + this.inputDestination = this.ctx.createMediaStreamDestination(); + this.analyser = this.ctx.createAnalyser(); + this.analyser.fftSize = 256; + + this.silentGain = this.ctx.createGain(); + this.silentGain.gain.value = 0; + + this.inputGain.connect(this.inputDestination); + this.inputGain.connect(this.analyser); + this.inputGain.connect(this.silentGain); + this.silentGain.connect(this.ctx.destination); + + this.inputGain.gain.setValueAtTime(1, this.ctx.currentTime); + + this.ctx.onstatechange = () => { + console.log(`[AudioManager] Context state: ${this.ctx?.state}`); + if (this.ctx?.state === 'running') { + this.notifyResumed(); + } + }; + + this.isInitialized = true; + } + + onResumed(cb: () => void) { + this.listeners.add(cb); + return () => this.listeners.delete(cb); + } + + private notifyResumed() { + this.listeners.forEach(cb => cb()); + } + + async resumeContext() { + if (!this.ctx) this.initContext(); + if (this.ctx && this.ctx.state === 'suspended') { + try { + await this.ctx.resume(); + console.log('[AudioManager] AudioContext resumed.'); + } catch (err) { + console.error('[AudioManager] Failed to resume context:', err); + } + } + } + + async setInputDevice(deviceId: string) { + if (!this.isInitialized) this.initContext(); + + // Skip if already set and stream is active + if (this.currentInputDeviceId === deviceId && this.currentStream?.active) { + return this.currentStream; + } + + try { + if (this.currentStream) { + this.currentStream.getTracks().forEach(t => t.stop()); + } + + const constraints = { + audio: { + deviceId: deviceId === 'default' ? undefined : { exact: deviceId }, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true + } + }; + + this.currentStream = await navigator.mediaDevices.getUserMedia(constraints); + this.currentInputDeviceId = deviceId; + + if (this.ctx && this.inputGain) { + if (this.inputSource) { + this.inputSource.disconnect(); + } + this.inputSource = this.ctx.createMediaStreamSource(this.currentStream); + this.inputSource.connect(this.inputGain); + } + + return this.currentStream; + } catch (err) { + console.error('[AudioManager] Failed to set input device:', err); + throw err; + } + } + + setInputVolume(volume: number) { + if (!this.isInitialized) this.initContext(); + if (this.inputGain && this.ctx) { + const gainValue = volume / 100; + this.inputGain.gain.setTargetAtTime(gainValue, this.ctx.currentTime, 0.1); + } + } + + /** + * CRITICAL: Always returns a CLONE of the destination track. + * This prevents LiveKit's cleanup from killing the main singleton track + * when switching rooms. + */ + getFreshTrack(): MediaStreamTrack | null { + if (!this.isInitialized) this.initContext(); + const track = this.inputDestination!.stream.getAudioTracks()[0]; + if (!track) return null; + return track.clone(); + } + + getAnalyserNode(): AnalyserNode { + if (!this.isInitialized) this.initContext(); + return this.analyser!; + } + + getContext(): AudioContext | null { + return this.ctx; + } +} diff --git a/packages/web/src/components/layout/AppLayout.tsx b/packages/web/src/components/layout/AppLayout.tsx index 31f11ae5..b38c2267 100644 --- a/packages/web/src/components/layout/AppLayout.tsx +++ b/packages/web/src/components/layout/AppLayout.tsx @@ -23,9 +23,37 @@ import { useServerStore } from '../../stores/serverStore'; import { useChatStore } from '../../stores/chatStore'; import { useUIStore } from '../../stores/uiStore'; import { useVoiceStore } from '../../stores/voiceStore'; +import { AudioManager } from '../../audio/AudioManager'; export function AppLayout() { const { serverId, channelId, inviteCode } = useParams<{ serverId?: string; channelId?: string; inviteCode?: string }>(); + + // Global interaction handler to resume AudioContext + useEffect(() => { + const resume = () => { + AudioManager.getInstance().resumeContext().then(() => { + // Wake up all audio/video elements that might be blocked by Autoplay + document.querySelectorAll('audio, video').forEach(el => { + (el as HTMLMediaElement).play().catch(() => { + // Silently fail if still blocked or no source + }); + }); + + window.removeEventListener('click', resume); + window.removeEventListener('keydown', resume); + window.removeEventListener('touchstart', resume); + }); + }; + window.addEventListener('click', resume); + window.addEventListener('keydown', resume); + window.addEventListener('touchstart', resume); + return () => { + window.removeEventListener('click', resume); + window.removeEventListener('keydown', resume); + window.removeEventListener('touchstart', resume); + }; + }, []); + const { user, isLoading } = useAuth(); const setCurrentServer = useServerStore((s) => s.setCurrentServer); const loadServerDetail = useServerStore((s) => s.loadServerDetail); diff --git a/packages/web/src/components/layout/ChannelSidebar.tsx b/packages/web/src/components/layout/ChannelSidebar.tsx index 2f8dd4d5..3a36dc1a 100644 --- a/packages/web/src/components/layout/ChannelSidebar.tsx +++ b/packages/web/src/components/layout/ChannelSidebar.tsx @@ -10,6 +10,7 @@ import { useVoiceStore } from '../../stores/voiceStore'; import { Avatar } from '../ui/Avatar'; import { wsSend } from '../../hooks/useWebSocket'; import { getActiveRoom } from '../../hooks/useLiveKit'; +import { AudioManager } from '../../audio/AudioManager'; export function ChannelSidebar() { const servers = useServerStore((s) => s.servers); @@ -31,14 +32,6 @@ export function ChannelSidebar() { const navigate = useNavigate(); const handleMicToggle = async () => { - const room = getActiveRoom(); - if (room) { - try { - await room.localParticipant.setMicrophoneEnabled(isMuted); - } catch (err) { - console.error('[ChannelSidebar] Failed to toggle mic:', err); - } - } toggleMic(); // Broadcast mute status via WebSocket so non-joined users can see it const willBeMuted = !isMuted; @@ -57,15 +50,6 @@ export function ChannelSidebar() { wsSend({ type: 'voice_status', isMuted: willBeMuted, isDeafened: willDeafen }); if (room) { try { - if (willDeafen) { - await room.localParticipant.setMicrophoneEnabled(false); - room.remoteParticipants.forEach((p) => p.setVolume(0)); - } else { - const outputVolume = useVoiceStore.getState().outputVolume; - const scaled = outputVolume / 100; - room.remoteParticipants.forEach((p) => p.setVolume(scaled)); - await room.localParticipant.setMicrophoneEnabled(true); - } // Broadcast deafen state to other participants via LiveKit data channel const encoder = new TextEncoder(); room.localParticipant.publishData( @@ -390,10 +374,14 @@ function UserAreaPanel({ const [openPanel, setOpenPanel] = useState<'input' | 'output' | null>(null); const [inputDevices, setInputDevices] = useState([]); const [outputDevices, setOutputDevices] = useState([]); - const [selectedInput, setSelectedInput] = useState('default'); - const [selectedOutput, setSelectedOutput] = useState('default'); + const inputDeviceId = useVoiceStore((s) => s.inputDeviceId); + const outputDeviceId = useVoiceStore((s) => s.outputDeviceId); + const setInputDevice = useVoiceStore((s) => s.setInputDevice); + const setOutputDevice = useVoiceStore((s) => s.setOutputDevice); + const [selectedInputLabel, setSelectedInputLabel] = useState('Default'); const [selectedOutputLabel, setSelectedOutputLabel] = useState('Default'); + const inputVolume = useVoiceStore((s) => s.inputVolume); const storeSetInputVolume = useVoiceStore((s) => s.setInputVolume); const outputVolume = useVoiceStore((s) => s.outputVolume); @@ -408,14 +396,24 @@ function UserAreaPanel({ const loadDevices = useCallback(async () => { try { // Need to request permission first to get labels - await navigator.mediaDevices.getUserMedia({ audio: true }).then(s => s.getTracks().forEach(t => t.stop())); + if (!AudioManager.getInstance().getContext()) { + await navigator.mediaDevices.getUserMedia({ audio: true }).then(s => s.getTracks().forEach(t => t.stop())); + } const devices = await navigator.mediaDevices.enumerateDevices(); - setInputDevices(devices.filter(d => d.kind === 'audioinput')); - setOutputDevices(devices.filter(d => d.kind === 'audiooutput')); + const inputs = devices.filter(d => d.kind === 'audioinput'); + const outputs = devices.filter(d => d.kind === 'audiooutput'); + setInputDevices(inputs); + setOutputDevices(outputs); + + const currentInput = inputs.find(d => d.deviceId === inputDeviceId); + if (currentInput) setSelectedInputLabel(currentInput.label || 'Default'); + + const currentOutput = outputs.find(d => d.deviceId === outputDeviceId); + if (currentOutput) setSelectedOutputLabel(currentOutput.label || 'Default'); } catch { // permission denied } - }, []); + }, [inputDeviceId, outputDeviceId]); // Start mic level monitoring when input panel opens useEffect(() => { @@ -425,17 +423,14 @@ function UserAreaPanel({ setMicLevel(0); return; } - let stream: MediaStream | null = null; - let ctx: AudioContext | null = null; + const start = async () => { try { - stream = await navigator.mediaDevices.getUserMedia({ audio: { deviceId: selectedInput !== 'default' ? selectedInput : undefined } }); - ctx = new AudioContext(); - const source = ctx.createMediaStreamSource(stream); - const analyser = ctx.createAnalyser(); + await AudioManager.getInstance().resumeContext(); + const analyser = AudioManager.getInstance().getAnalyserNode(); analyser.fftSize = 256; - source.connect(analyser); analyserRef.current = analyser; + const data = new Uint8Array(analyser.frequencyBinCount); const tick = () => { if (!analyserRef.current) return; @@ -451,10 +446,8 @@ function UserAreaPanel({ return () => { if (animFrameRef.current) cancelAnimationFrame(animFrameRef.current); analyserRef.current = null; - stream?.getTracks().forEach(t => t.stop()); - ctx?.close(); }; - }, [openPanel, selectedInput]); + }, [openPanel]); useEffect(() => { const handleClick = (e: MouseEvent) => { @@ -476,19 +469,19 @@ function UserAreaPanel({ setOpenPanel(panel); setShowInputDeviceList(false); setShowOutputDeviceList(false); + // Explicitly resume on interaction + AudioManager.getInstance().resumeContext(); } }; const selectInput = (device: MediaDeviceInfo) => { - setSelectedInput(device.deviceId); + setInputDevice(device.deviceId); setSelectedInputLabel(device.label || 'Default'); setShowInputDeviceList(false); - const room = getActiveRoom(); - if (room) room.switchActiveDevice('audioinput', device.deviceId).catch(() => {}); }; const selectOutput = (device: MediaDeviceInfo) => { - setSelectedOutput(device.deviceId); + setOutputDevice(device.deviceId); setSelectedOutputLabel(device.label || 'Default'); setShowOutputDeviceList(false); const room = getActiveRoom(); @@ -525,15 +518,15 @@ function UserAreaPanel({ key={d.deviceId} onClick={() => selectInput(d)} className={`w-full px-3 py-2 text-left text-[13px] hover:bg-discord-modifier-hover transition-colors flex items-center gap-2 ${ - selectedInput === d.deviceId ? 'text-discord-text-primary' : 'text-discord-text-secondary' + inputDeviceId === d.deviceId ? 'text-discord-text-primary' : 'text-discord-text-secondary' }`} > - {selectedInput === d.deviceId && ( + {inputDeviceId === d.deviceId && ( )} - {d.label || 'Default'} + {d.label || 'Default'} ))} @@ -550,22 +543,11 @@ function UserAreaPanel({ min={0} max={200} value={inputVolume} - onChange={(e) => { - const vol = Number(e.target.value); - storeSetInputVolume(vol); - - const room = getActiveRoom(); - if (room) { - const { isMuted: manuallyMuted, isDeafened: manuallyDeafened } = useVoiceStore.getState(); - // If user is manually muted, hardware should stay off regardless of volume. - // If user is NOT manually muted and volume is 0, we can keep hardware ON - // (Web Audio handles silence) or turn it OFF for battery/privacy. - // Discord keeps it ON (green ring) but silent. We'll follow that. - if (!manuallyMuted && !manuallyDeafened && !room.localParticipant.isMicrophoneEnabled && vol > 0) { - room.localParticipant.setMicrophoneEnabled(true).catch(() => {}); - } - } - }} className="w-full h-1.5 rounded-full appearance-none cursor-pointer accent-discord-blurple bg-discord-bg-tertiary [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:w-3 [&::-webkit-slider-thumb]:h-3 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-white [&::-webkit-slider-thumb]:shadow-md" + onChange={(e) => { + const vol = Number(e.target.value); + storeSetInputVolume(vol); + }} + className="w-full h-1.5 rounded-full appearance-none cursor-pointer accent-discord-blurple bg-discord-bg-tertiary [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:w-3 [&::-webkit-slider-thumb]:h-3 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-white [&::-webkit-slider-thumb]:shadow-md" style={{ background: `linear-gradient(to right, #5865f2 0%, #5865f2 ${inputVolume / 2}%, #4e5058 ${inputVolume / 2}%, #4e5058 100%)`, }} @@ -622,15 +604,15 @@ function UserAreaPanel({ key={d.deviceId} onClick={() => selectOutput(d)} className={`w-full px-3 py-2 text-left text-[13px] hover:bg-discord-modifier-hover transition-colors flex items-center gap-2 ${ - selectedOutput === d.deviceId ? 'text-discord-text-primary' : 'text-discord-text-secondary' + outputDeviceId === d.deviceId ? 'text-discord-text-primary' : 'text-discord-text-secondary' }`} > - {selectedOutput === d.deviceId && ( + {outputDeviceId === d.deviceId && ( )} - {d.label || 'Default'} + {d.label || 'Default'} ))} diff --git a/packages/web/src/components/voice/VoiceControlBar.tsx b/packages/web/src/components/voice/VoiceControlBar.tsx index 737e2106..de298a7c 100644 --- a/packages/web/src/components/voice/VoiceControlBar.tsx +++ b/packages/web/src/components/voice/VoiceControlBar.tsx @@ -36,14 +36,6 @@ export function VoiceControlBar() { const [qualityOpen, setQualityOpen] = useState(false); const handleMute = React.useCallback(async () => { - const room = getActiveRoom(); - if (room) { - try { - await room.localParticipant.setMicrophoneEnabled(isMuted); - } catch (err) { - console.error('[VoiceControlBar] Failed to toggle mic:', err); - } - } toggleMic(); // Broadcast via WebSocket so sidebar shows status without joining wsSend({ type: 'voice_status', isMuted: !isMuted, isDeafened }); @@ -60,14 +52,6 @@ export function VoiceControlBar() { wsSend({ type: 'voice_status', isMuted: willDeafen, isDeafened: willDeafen }); if (room) { try { - if (willDeafen) { - await room.localParticipant.setMicrophoneEnabled(false); - room.remoteParticipants.forEach((p) => p.setVolume(0)); - } else { - const outputVolume = useVoiceStore.getState().outputVolume; - room.remoteParticipants.forEach((p) => p.setVolume(outputVolume / 100)); - await room.localParticipant.setMicrophoneEnabled(true); - } // Broadcast deafen state via LiveKit data channel for in-room users const encoder = new TextEncoder(); room.localParticipant.publishData( diff --git a/packages/web/src/components/voice/VoiceUser.tsx b/packages/web/src/components/voice/VoiceUser.tsx index 27706756..4c45248f 100644 --- a/packages/web/src/components/voice/VoiceUser.tsx +++ b/packages/web/src/components/voice/VoiceUser.tsx @@ -1,7 +1,7 @@ import React, { useRef, useEffect, useState, useCallback } from 'react'; import { Avatar } from '../ui/Avatar'; import { useVoiceStore } from '../../stores/voiceStore'; -import { getSharedAudioCtx } from '../../hooks/useLiveKit'; +import { AudioManager } from '../../audio/AudioManager'; import type { ParticipantInfo } from '../../hooks/useLiveKit'; interface VoiceUserProps { @@ -12,100 +12,132 @@ interface VoiceUserProps { export function VoiceUser({ participant, large }: VoiceUserProps) { const videoRef = useRef(null); const audioRef = useRef(null); + const isDeafened = useVoiceStore((s) => s.isDeafened); const outputVolume = useVoiceStore((s) => s.outputVolume); const participantVolumes = useVoiceStore((s) => s.participantVolumes); + const [, forceUpdate] = useState(0); const perUserVolume = participantVolumes.get(participant.userId) ?? 100; const isLocal = participant.isLocal; - // Web Audio for volume boost (> 100%) - const gainNodeRef = useRef(null); - const sourceNodeRef = useRef(null); + // --- AUDIO PIPELINE: NATIVE FIRST --- + + // Refs for the optional boost pipeline + const boostGainRef = useRef(null); + const boostSourceRef = useRef(null); + + // 1. Basic Track Attachment (The Rock-Solid Foundation) + useEffect(() => { + const audioEl = audioRef.current; + if (isLocal || !audioEl || !participant.audioTrack) return; + + // Direct attachment. + const stream = new MediaStream([participant.audioTrack]); + + // Only update if changed to prevent interruptions + if ((audioEl.srcObject as MediaStream)?.id !== stream.id) { + audioEl.srcObject = stream; + + // Aggressive play attempt for Chrome + const tryPlay = async () => { + try { + await audioEl.play(); + } catch (err) { + console.warn("[Audio] Autoplay blocked, retrying...", err); + // If blocked, we rely on the global interaction listener to resume context, + // but we can also retry play() on the element itself on next click. + } + }; + tryPlay(); + } + }, [participant.audioTrack, isLocal]); + + // 2. Volume Management (Hybrid) + useEffect(() => { + const audioEl = audioRef.current; + if (isLocal || !audioEl || !participant.audioTrack) return; + + const globalScale = outputVolume / 100; + const userScale = perUserVolume / 100; + const finalVolume = globalScale * userScale; + + if (isDeafened) { + audioEl.muted = true; + return; + } + + // Logic: + // If we are boosting (>100%) AND context is running, use Web Audio. + // Otherwise, stick to the native element for maximum reliability. + + const audioManager = AudioManager.getInstance(); + const ctx = audioManager.getContext(); + const isBoosting = finalVolume > 1.0; + const isContextReady = ctx && ctx.state === 'running'; + + if (isBoosting && isContextReady) { + // --- BOOST MODE (>100%) --- + // Setup pipeline if missing + if (!boostGainRef.current && ctx) { + const gain = ctx.createGain(); + const source = ctx.createMediaStreamSource(new MediaStream([participant.audioTrack])); + + source.connect(gain); + gain.connect(ctx.destination); + + boostGainRef.current = gain; + boostSourceRef.current = source; + } + + // Apply boosted gain + if (boostGainRef.current && ctx) { + boostGainRef.current.gain.setTargetAtTime(finalVolume, ctx.currentTime, 0.01); + } + + // MUTE the element so we don't double audio + audioEl.muted = true; + + } else { + // --- STANDARD MODE (0% - 100%) --- + // Clean up boost pipeline if it exists + if (boostSourceRef.current) { + boostSourceRef.current.disconnect(); + boostSourceRef.current = null; + boostGainRef.current = null; + } + + // Use the element + audioEl.muted = false; + audioEl.volume = Math.min(finalVolume, 1.0); + + // Ensure it's playing (in case it was paused/blocked earlier) + if (audioEl.paused) { + audioEl.play().catch(() => {}); + } + } + }, [outputVolume, perUserVolume, isDeafened, isLocal, participant.audioTrack]); + + + // --- VIDEO & UI --- - // Determine active video track const liveScreen = participant.isScreenSharing && participant.screenTrack?.readyState === 'live' ? participant.screenTrack : null; const liveCamera = participant.isCameraOn && participant.videoTrack?.readyState === 'live' ? participant.videoTrack : null; const activeVideoTrack = liveScreen ?? liveCamera; const hasVideo = activeVideoTrack !== null; const isScreenShare = liveScreen !== null; - // 1. STANDARD AUDIO PLAYBACK (Reliability Layer) + // Force re-render when tracks end/mute useEffect(() => { - const audioEl = audioRef.current; - if (isLocal || !audioEl || !participant.audioTrack) return; + const tracks = [participant.videoTrack, participant.screenTrack].filter((t): t is MediaStreamTrack => t !== null); + if (tracks.length === 0) return; + const onEnded = () => forceUpdate((n) => n + 1); + tracks.forEach((t) => t.addEventListener('ended', onEnded)); + return () => tracks.forEach((t) => t.removeEventListener('ended', onEnded)); + }, [participant.videoTrack, participant.screenTrack]); - const stream = new MediaStream([participant.audioTrack]); - if ((audioEl.srcObject as MediaStream)?.id !== stream.id) { - audioEl.srcObject = stream; - // Critical for Chrome: Explicitly call play() - audioEl.play().catch((err) => console.warn('[Audio] Auto-play blocked:', err)); - } - }, [participant.audioTrack, isLocal]); - - // 2. VOLUME & BOOST CONTROL - useEffect(() => { - const audioEl = audioRef.current; - if (isLocal || !audioEl || !participant.audioTrack) return; - - // Calculate total requested volume (0.0 to 2.0+) - const combined = (perUserVolume / 100) * (outputVolume / 100); - - if (isDeafened) { - audioEl.muted = true; - if (gainNodeRef.current) gainNodeRef.current.gain.value = 0; - return; - } - - // Logic: - // 0% - 100%: Use standard