devsyncwrld/backspace
1
0
Fork 0
Code Issues Pull Requests Actions 26 Packages Projects Releases Wiki Activity

Overhaul audio architecture for robust cross-browser reliability

Browse Source
This commit is contained in:
Jannis Braun
2026-02-19 22:13:48 +01:00
parent 8afd960a43
commit 95217f0343
7 changed files with 439 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/web/src/components/layout/AppLayout.tsx
+28
View File
@@ -23,9 +23,37 @@ import { useServerStore } from '../../stores/serverStore';
import { useChatStore } from '../../stores/chatStore';
import { useUIStore } from '../../stores/uiStore';
import { useVoiceStore } from '../../stores/voiceStore';
import { AudioManager } from '../../audio/AudioManager';
export function AppLayout() {
const { serverId, channelId, inviteCode } = useParams<{ serverId?: string; channelId?: string; inviteCode?: string }>();
// Global interaction handler to resume AudioContext
useEffect(() => {
const resume = () => {
AudioManager.getInstance().resumeContext().then(() => {
// Wake up all audio/video elements that might be blocked by Autoplay
document.querySelectorAll('audio, video').forEach(el => {
(el as HTMLMediaElement).play().catch(() => {
// Silently fail if still blocked or no source
});
});
window.removeEventListener('click', resume);
window.removeEventListener('keydown', resume);
window.removeEventListener('touchstart', resume);
});
};
window.addEventListener('click', resume);
window.addEventListener('keydown', resume);
window.addEventListener('touchstart', resume);
return () => {
window.removeEventListener('click', resume);
window.removeEventListener('keydown', resume);
window.removeEventListener('touchstart', resume);
};
}, []);
const { user, isLoading } = useAuth();
const setCurrentServer = useServerStore((s) => s.setCurrentServer);
const loadServerDetail = useServerStore((s) => s.loadServerDetail);
packages/web/src/components/layout/ChannelSidebar.tsx
+41 -59
View File
@@ -10,6 +10,7 @@ import { useVoiceStore } from '../../stores/voiceStore';
import { Avatar } from '../ui/Avatar';
import { wsSend } from '../../hooks/useWebSocket';
import { getActiveRoom } from '../../hooks/useLiveKit';
import { AudioManager } from '../../audio/AudioManager';
export function ChannelSidebar() {
const servers = useServerStore((s) => s.servers);
@@ -31,14 +32,6 @@ export function ChannelSidebar() {
const navigate = useNavigate();
const handleMicToggle = async () => {
const room = getActiveRoom();
if (room) {
try {
await room.localParticipant.setMicrophoneEnabled(isMuted);
} catch (err) {
console.error('[ChannelSidebar] Failed to toggle mic:', err);
}
}
toggleMic();
// Broadcast mute status via WebSocket so non-joined users can see it
const willBeMuted = !isMuted;
@@ -57,15 +50,6 @@ export function ChannelSidebar() {
wsSend({ type: 'voice_status', isMuted: willBeMuted, isDeafened: willDeafen });
if (room) {
try {
if (willDeafen) {
await room.localParticipant.setMicrophoneEnabled(false);
room.remoteParticipants.forEach((p) => p.setVolume(0));
} else {
const outputVolume = useVoiceStore.getState().outputVolume;
const scaled = outputVolume / 100;
room.remoteParticipants.forEach((p) => p.setVolume(scaled));
await room.localParticipant.setMicrophoneEnabled(true);
}
// Broadcast deafen state to other participants via LiveKit data channel
const encoder = new TextEncoder();
room.localParticipant.publishData(
@@ -390,10 +374,14 @@ function UserAreaPanel({
const [openPanel, setOpenPanel] = useState<'input' | 'output' | null>(null);
const [inputDevices, setInputDevices] = useState<MediaDeviceInfo[]>([]);
const [outputDevices, setOutputDevices] = useState<MediaDeviceInfo[]>([]);
const [selectedInput, setSelectedInput] = useState<string>('default');
const [selectedOutput, setSelectedOutput] = useState<string>('default');
const inputDeviceId = useVoiceStore((s) => s.inputDeviceId);
const outputDeviceId = useVoiceStore((s) => s.outputDeviceId);
const setInputDevice = useVoiceStore((s) => s.setInputDevice);
const setOutputDevice = useVoiceStore((s) => s.setOutputDevice);
const [selectedInputLabel, setSelectedInputLabel] = useState<string>('Default');
const [selectedOutputLabel, setSelectedOutputLabel] = useState<string>('Default');
const inputVolume = useVoiceStore((s) => s.inputVolume);
const storeSetInputVolume = useVoiceStore((s) => s.setInputVolume);
const outputVolume = useVoiceStore((s) => s.outputVolume);
@@ -408,14 +396,24 @@ function UserAreaPanel({
const loadDevices = useCallback(async () => {
try {
// Need to request permission first to get labels
await navigator.mediaDevices.getUserMedia({ audio: true }).then(s => s.getTracks().forEach(t => t.stop()));
if (!AudioManager.getInstance().getContext()) {
await navigator.mediaDevices.getUserMedia({ audio: true }).then(s => s.getTracks().forEach(t => t.stop()));
}
const devices = await navigator.mediaDevices.enumerateDevices();
setInputDevices(devices.filter(d => d.kind === 'audioinput'));
setOutputDevices(devices.filter(d => d.kind === 'audiooutput'));
const inputs = devices.filter(d => d.kind === 'audioinput');
const outputs = devices.filter(d => d.kind === 'audiooutput');
setInputDevices(inputs);
setOutputDevices(outputs);
const currentInput = inputs.find(d => d.deviceId === inputDeviceId);
if (currentInput) setSelectedInputLabel(currentInput.label || 'Default');
const currentOutput = outputs.find(d => d.deviceId === outputDeviceId);
if (currentOutput) setSelectedOutputLabel(currentOutput.label || 'Default');
} catch {
// permission denied
}
}, []);
}, [inputDeviceId, outputDeviceId]);
// Start mic level monitoring when input panel opens
useEffect(() => {
@@ -425,17 +423,14 @@ function UserAreaPanel({
setMicLevel(0);
return;
}
let stream: MediaStream | null = null;
let ctx: AudioContext | null = null;
const start = async () => {
try {
stream = await navigator.mediaDevices.getUserMedia({ audio: { deviceId: selectedInput !== 'default' ? selectedInput : undefined } });
ctx = new AudioContext();
const source = ctx.createMediaStreamSource(stream);
const analyser = ctx.createAnalyser();
await AudioManager.getInstance().resumeContext();
const analyser = AudioManager.getInstance().getAnalyserNode();
analyser.fftSize = 256;
source.connect(analyser);
analyserRef.current = analyser;
const data = new Uint8Array(analyser.frequencyBinCount);
const tick = () => {
if (!analyserRef.current) return;
@@ -451,10 +446,8 @@ function UserAreaPanel({
return () => {
if (animFrameRef.current) cancelAnimationFrame(animFrameRef.current);
analyserRef.current = null;
stream?.getTracks().forEach(t => t.stop());
ctx?.close();
};
}, [openPanel, selectedInput]);
}, [openPanel]);
useEffect(() => {
const handleClick = (e: MouseEvent) => {
@@ -476,19 +469,19 @@ function UserAreaPanel({
setOpenPanel(panel);
setShowInputDeviceList(false);
setShowOutputDeviceList(false);
// Explicitly resume on interaction
AudioManager.getInstance().resumeContext();
}
};
const selectInput = (device: MediaDeviceInfo) => {
setSelectedInput(device.deviceId);
setInputDevice(device.deviceId);
setSelectedInputLabel(device.label || 'Default');
setShowInputDeviceList(false);
const room = getActiveRoom();
if (room) room.switchActiveDevice('audioinput', device.deviceId).catch(() => {});
};
const selectOutput = (device: MediaDeviceInfo) => {
setSelectedOutput(device.deviceId);
setOutputDevice(device.deviceId);
setSelectedOutputLabel(device.label || 'Default');
setShowOutputDeviceList(false);
const room = getActiveRoom();
@@ -525,15 +518,15 @@ function UserAreaPanel({
key={d.deviceId}
onClick={() => selectInput(d)}
className={`w-full px-3 py-2 text-left text-[13px] hover:bg-discord-modifier-hover transition-colors flex items-center gap-2 ${
selectedInput === d.deviceId ? 'text-discord-text-primary' : 'text-discord-text-secondary'
inputDeviceId === d.deviceId ? 'text-discord-text-primary' : 'text-discord-text-secondary'
}`}
>
{selectedInput === d.deviceId && (
{inputDeviceId === d.deviceId && (
<svg width="16" height="16" viewBox="0 0 24 24" fill="currentColor" className="text-discord-blurple flex-shrink-0">
<path d="M9 16.17L4.83 12l-1.42 1.41L9 19 21 7l-1.41-1.41z" />
</svg>
)}
<span className={selectedInput === d.deviceId ? '' : 'pl-6'}>{d.label || 'Default'}</span>
<span className={inputDeviceId === d.deviceId ? '' : 'pl-6'}>{d.label || 'Default'}</span>
</button>
))}
</div>
@@ -550,22 +543,11 @@ function UserAreaPanel({
min={0}
max={200}
value={inputVolume}
onChange={(e) => {
const vol = Number(e.target.value);
storeSetInputVolume(vol);
const room = getActiveRoom();
if (room) {
const { isMuted: manuallyMuted, isDeafened: manuallyDeafened } = useVoiceStore.getState();
// If user is manually muted, hardware should stay off regardless of volume.
// If user is NOT manually muted and volume is 0, we can keep hardware ON
// (Web Audio handles silence) or turn it OFF for battery/privacy.
// Discord keeps it ON (green ring) but silent. We'll follow that.
if (!manuallyMuted && !manuallyDeafened && !room.localParticipant.isMicrophoneEnabled && vol > 0) {
room.localParticipant.setMicrophoneEnabled(true).catch(() => {});
}
}
}} className="w-full h-1.5 rounded-full appearance-none cursor-pointer accent-discord-blurple bg-discord-bg-tertiary [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:w-3 [&::-webkit-slider-thumb]:h-3 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-white [&::-webkit-slider-thumb]:shadow-md"
onChange={(e) => {
const vol = Number(e.target.value);
storeSetInputVolume(vol);
}}
className="w-full h-1.5 rounded-full appearance-none cursor-pointer accent-discord-blurple bg-discord-bg-tertiary [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:w-3 [&::-webkit-slider-thumb]:h-3 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-white [&::-webkit-slider-thumb]:shadow-md"
style={{
background: `linear-gradient(to right, #5865f2 0%, #5865f2 ${inputVolume / 2}%, #4e5058 ${inputVolume / 2}%, #4e5058 100%)`,
}}
@@ -622,15 +604,15 @@ function UserAreaPanel({
key={d.deviceId}
onClick={() => selectOutput(d)}
className={`w-full px-3 py-2 text-left text-[13px] hover:bg-discord-modifier-hover transition-colors flex items-center gap-2 ${
selectedOutput === d.deviceId ? 'text-discord-text-primary' : 'text-discord-text-secondary'
outputDeviceId === d.deviceId ? 'text-discord-text-primary' : 'text-discord-text-secondary'
}`}
>
{selectedOutput === d.deviceId && (
{outputDeviceId === d.deviceId && (
<svg width="16" height="16" viewBox="0 0 24 24" fill="currentColor" className="text-discord-blurple flex-shrink-0">
<path d="M9 16.17L4.83 12l-1.42 1.41L9 19 21 7l-1.41-1.41z" />
</svg>
)}
<span className={selectedOutput === d.deviceId ? '' : 'pl-6'}>{d.label || 'Default'}</span>
<span className={outputDeviceId === d.deviceId ? '' : 'pl-6'}>{d.label || 'Default'}</span>
</button>
))}
</div>
packages/web/src/components/voice/VoiceControlBar.tsx
-16
View File
@@ -36,14 +36,6 @@ export function VoiceControlBar() {
const [qualityOpen, setQualityOpen] = useState(false);
const handleMute = React.useCallback(async () => {
const room = getActiveRoom();
if (room) {
try {
await room.localParticipant.setMicrophoneEnabled(isMuted);
} catch (err) {
console.error('[VoiceControlBar] Failed to toggle mic:', err);
}
}
toggleMic();
// Broadcast via WebSocket so sidebar shows status without joining
wsSend({ type: 'voice_status', isMuted: !isMuted, isDeafened });
@@ -60,14 +52,6 @@ export function VoiceControlBar() {
wsSend({ type: 'voice_status', isMuted: willDeafen, isDeafened: willDeafen });
if (room) {
try {
if (willDeafen) {
await room.localParticipant.setMicrophoneEnabled(false);
room.remoteParticipants.forEach((p) => p.setVolume(0));
} else {
const outputVolume = useVoiceStore.getState().outputVolume;
room.remoteParticipants.forEach((p) => p.setVolume(outputVolume / 100));
await room.localParticipant.setMicrophoneEnabled(true);
}
// Broadcast deafen state via LiveKit data channel for in-room users
const encoder = new TextEncoder();
room.localParticipant.publishData(
packages/web/src/components/voice/VoiceUser.tsx
+121 -113
View File
@@ -1,7 +1,7 @@
import React, { useRef, useEffect, useState, useCallback } from 'react';
import { Avatar } from '../ui/Avatar';
import { useVoiceStore } from '../../stores/voiceStore';
import { getSharedAudioCtx } from '../../hooks/useLiveKit';
import { AudioManager } from '../../audio/AudioManager';
import type { ParticipantInfo } from '../../hooks/useLiveKit';
interface VoiceUserProps {
@@ -12,100 +12,132 @@ interface VoiceUserProps {
export function VoiceUser({ participant, large }: VoiceUserProps) {
const videoRef = useRef<HTMLVideoElement>(null);
const audioRef = useRef<HTMLAudioElement>(null);
const isDeafened = useVoiceStore((s) => s.isDeafened);
const outputVolume = useVoiceStore((s) => s.outputVolume);
const participantVolumes = useVoiceStore((s) => s.participantVolumes);
const [, forceUpdate] = useState(0);
const perUserVolume = participantVolumes.get(participant.userId) ?? 100;
const isLocal = participant.isLocal;
// Web Audio for volume boost (> 100%)
const gainNodeRef = useRef<GainNode | null>(null);
const sourceNodeRef = useRef<MediaStreamAudioSourceNode | null>(null);
// --- AUDIO PIPELINE: NATIVE FIRST ---
// Refs for the optional boost pipeline
const boostGainRef = useRef<GainNode | null>(null);
const boostSourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
// 1. Basic Track Attachment (The Rock-Solid Foundation)
useEffect(() => {
const audioEl = audioRef.current;
if (isLocal || !audioEl || !participant.audioTrack) return;
// Direct attachment.
const stream = new MediaStream([participant.audioTrack]);
// Only update if changed to prevent interruptions
if ((audioEl.srcObject as MediaStream)?.id !== stream.id) {
audioEl.srcObject = stream;
// Aggressive play attempt for Chrome
const tryPlay = async () => {
try {
await audioEl.play();
} catch (err) {
console.warn("[Audio] Autoplay blocked, retrying...", err);
// If blocked, we rely on the global interaction listener to resume context,
// but we can also retry play() on the element itself on next click.
}
};
tryPlay();
}
}, [participant.audioTrack, isLocal]);
// 2. Volume Management (Hybrid)
useEffect(() => {
const audioEl = audioRef.current;
if (isLocal || !audioEl || !participant.audioTrack) return;
const globalScale = outputVolume / 100;
const userScale = perUserVolume / 100;
const finalVolume = globalScale * userScale;
if (isDeafened) {
audioEl.muted = true;
return;
}
// Logic:
// If we are boosting (>100%) AND context is running, use Web Audio.
// Otherwise, stick to the native element for maximum reliability.
const audioManager = AudioManager.getInstance();
const ctx = audioManager.getContext();
const isBoosting = finalVolume > 1.0;
const isContextReady = ctx && ctx.state === 'running';
if (isBoosting && isContextReady) {
// --- BOOST MODE (>100%) ---
// Setup pipeline if missing
if (!boostGainRef.current && ctx) {
const gain = ctx.createGain();
const source = ctx.createMediaStreamSource(new MediaStream([participant.audioTrack]));
source.connect(gain);
gain.connect(ctx.destination);
boostGainRef.current = gain;
boostSourceRef.current = source;
}
// Apply boosted gain
if (boostGainRef.current && ctx) {
boostGainRef.current.gain.setTargetAtTime(finalVolume, ctx.currentTime, 0.01);
}
// MUTE the element so we don't double audio
audioEl.muted = true;
} else {
// --- STANDARD MODE (0% - 100%) ---
// Clean up boost pipeline if it exists
if (boostSourceRef.current) {
boostSourceRef.current.disconnect();
boostSourceRef.current = null;
boostGainRef.current = null;
}
// Use the element
audioEl.muted = false;
audioEl.volume = Math.min(finalVolume, 1.0);
// Ensure it's playing (in case it was paused/blocked earlier)
if (audioEl.paused) {
audioEl.play().catch(() => {});
}
}
}, [outputVolume, perUserVolume, isDeafened, isLocal, participant.audioTrack]);
// --- VIDEO & UI ---
// Determine active video track
const liveScreen = participant.isScreenSharing && participant.screenTrack?.readyState === 'live' ? participant.screenTrack : null;
const liveCamera = participant.isCameraOn && participant.videoTrack?.readyState === 'live' ? participant.videoTrack : null;
const activeVideoTrack = liveScreen ?? liveCamera;
const hasVideo = activeVideoTrack !== null;
const isScreenShare = liveScreen !== null;
// 1. STANDARD AUDIO PLAYBACK (Reliability Layer)
// Force re-render when tracks end/mute
useEffect(() => {
const audioEl = audioRef.current;
if (isLocal || !audioEl || !participant.audioTrack) return;
const tracks = [participant.videoTrack, participant.screenTrack].filter((t): t is MediaStreamTrack => t !== null);
if (tracks.length === 0) return;
const onEnded = () => forceUpdate((n) => n + 1);
tracks.forEach((t) => t.addEventListener('ended', onEnded));
return () => tracks.forEach((t) => t.removeEventListener('ended', onEnded));
}, [participant.videoTrack, participant.screenTrack]);
const stream = new MediaStream([participant.audioTrack]);
if ((audioEl.srcObject as MediaStream)?.id !== stream.id) {
audioEl.srcObject = stream;
// Critical for Chrome: Explicitly call play()
audioEl.play().catch((err) => console.warn('[Audio] Auto-play blocked:', err));
}
}, [participant.audioTrack, isLocal]);
// 2. VOLUME & BOOST CONTROL
useEffect(() => {
const audioEl = audioRef.current;
if (isLocal || !audioEl || !participant.audioTrack) return;
// Calculate total requested volume (0.0 to 2.0+)
const combined = (perUserVolume / 100) * (outputVolume / 100);
if (isDeafened) {
audioEl.muted = true;
if (gainNodeRef.current) gainNodeRef.current.gain.value = 0;
return;
}
// Logic:
// 0% - 100%: Use standard <audio> volume. Disconnect Web Audio to prevent doubling.
// > 100%: Set <audio> to 100%, connect Web Audio for the EXTRA boost.
// Standard Path (Always Active unless >100% needs to take over completely, but doubling is risk.
// SAFE APPROACH: Use <audio> for everything up to 100%.
// If > 100%, keep <audio> at 100% and add Web Audio *parallel*? No, that causes phasing.
// CORRECT APPROACH:
// If <= 100%: Element Volume = combined. Web Audio = Disconnected.
// If > 100%: Element Volume = 0 (Muted). Web Audio = connected & combined.
const ctx = getSharedAudioCtx();
const useWebAudio = combined > 1.0 && ctx && ctx.state === 'running';
if (useWebAudio) {
// --- BOOST MODE (>100%) ---
// Mute standard element to prevent double audio
audioEl.muted = true;
// Setup/Connect Web Audio
if (!gainNodeRef.current) {
gainNodeRef.current = ctx.createGain();
gainNodeRef.current.connect(ctx.destination);
}
if (!sourceNodeRef.current) {
sourceNodeRef.current = ctx.createMediaStreamSource(new MediaStream([participant.audioTrack]));
sourceNodeRef.current.connect(gainNodeRef.current);
}
// Apply full gain (e.g., 1.5, 2.0)
gainNodeRef.current.gain.setTargetAtTime(combined, ctx.currentTime, 0.01);
} else {
// --- STANDARD MODE (0-100%) ---
// Cleanup Web Audio to prevent doubling/leaking
if (sourceNodeRef.current) {
sourceNodeRef.current.disconnect();
sourceNodeRef.current = null;
}
// Use standard element
audioEl.muted = false;
audioEl.volume = Math.min(combined, 1.0);
}
}, [isDeafened, outputVolume, perUserVolume, isLocal, participant.audioTrack]);
// Video Handling
// Attach Video
useEffect(() => {
const videoEl = videoRef.current;
if (!videoEl) return;
@@ -116,32 +148,15 @@ export function VoiceUser({ participant, large }: VoiceUserProps) {
}
}, [activeVideoTrack]);
// Cleanup Listeners
useEffect(() => {
const tracks = [participant.videoTrack, participant.screenTrack].filter((t): t is MediaStreamTrack => t !== null);
if (tracks.length === 0) return;
const onEnded = () => forceUpdate((n) => n + 1);
tracks.forEach((t) => t.addEventListener('ended', onEnded));
return () => tracks.forEach((t) => t.removeEventListener('ended', onEnded));
}, [participant.videoTrack, participant.screenTrack]);
// Interaction (Resume Context)
const handleInteraction = useCallback(() => {
const ctx = getSharedAudioCtx();
if (ctx && ctx.state === 'suspended') {
ctx.resume().catch(console.error);
}
}, []);
const setParticipantVolume = useVoiceStore((s) => s.setParticipantVolume);
// Context Menu
const [volumeMenu, setVolumeMenu] = useState<{ x: number; y: number } | null>(null);
const setParticipantVolume = useVoiceStore((s) => s.setParticipantVolume);
const handleContextMenu = useCallback((e: React.MouseEvent) => {
if (isLocal) return;
e.preventDefault();
handleInteraction();
setVolumeMenu({ x: e.clientX, y: e.clientY });
}, [isLocal, handleInteraction]);
}, [isLocal]);
useEffect(() => {
if (!volumeMenu) return;
@@ -152,7 +167,6 @@ export function VoiceUser({ participant, large }: VoiceUserProps) {
return (
<div
onClick={handleInteraction}
className={`relative bg-[#111214] rounded-xl overflow-hidden flex items-center justify-center group transition-all duration-200 ${
participant.isSpeaking
? 'ring-[3px] ring-discord-green shadow-[0_0_12px_rgba(35,165,90,0.25)]'
@@ -160,7 +174,11 @@ export function VoiceUser({ participant, large }: VoiceUserProps) {
} ${large ? 'h-full w-full' : 'h-full aspect-video'}`}
onContextMenu={handleContextMenu}
>
{/* Audio Element: Primary playback device */}
{/*
Native Audio Element
- AutoPlay is critical
- PlaysInline is critical for mobile
*/}
{!isLocal && <audio ref={audioRef} autoPlay playsInline />}
{hasVideo ? (
@@ -227,13 +245,7 @@ export function VoiceUser({ participant, large }: VoiceUserProps) {
User Volume
</div>
<div className="flex items-center gap-2">
<svg
width="16"
height="16"
viewBox="0 0 24 24"
fill="currentColor"
className="text-discord-text-muted flex-shrink-0"
>
<svg width="16" height="16" viewBox="0 0 24 24" fill="currentColor" className="text-discord-text-muted flex-shrink-0">
<path d="M3 9v6h4l5 5V4L7 9H3z" />
</svg>
<input
@@ -241,14 +253,10 @@ export function VoiceUser({ participant, large }: VoiceUserProps) {
min="0"
max="200"
value={perUserVolume}
onChange={(e) =>
setParticipantVolume(participant.userId, parseInt(e.target.value))
}
onChange={(e) => setParticipantVolume(participant.userId, parseInt(e.target.value))}
className="flex-1 accent-discord-blurple h-1"
/>
<span className="text-xs text-discord-text-secondary min-w-[32px] text-right">
{perUserVolume}%
</span>
<span className="text-xs text-discord-text-secondary min-w-[32px] text-right">{perUserVolume}%</span>
</div>
</div>
)}