devsyncwrld/backspace
1
0
Fork 0
Code Issues Pull Requests Actions 25 Packages Projects Releases Wiki Activity

Overhaul audio architecture for robust cross-browser reliability

Browse Source
This commit is contained in:
Jannis Braun
2026-02-19 22:13:48 +01:00
parent 8afd960a43
commit 95217f0343
7 changed files with 439 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/web/src/audio/AudioManager.ts
+144
View File
@@ -0,0 +1,144 @@
export class AudioManager {
private static instance: AudioManager | null = null;
private ctx: AudioContext | null = null;
private inputGain: GainNode | null = null;
private inputSource: MediaStreamAudioSourceNode | null = null;
private inputDestination: MediaStreamAudioDestinationNode | null = null;
private silentGain: GainNode | null = null;
private analyser: AnalyserNode | null = null;
private currentInputDeviceId: string = 'default';
private currentStream: MediaStream | null = null;
private isInitialized = false;
private listeners: Set<() => void> = new Set();
private constructor() {}
static getInstance(): AudioManager {
if (!AudioManager.instance) {
AudioManager.instance = new AudioManager();
}
return AudioManager.instance;
}
private initContext() {
if (this.ctx) return;
const AudioContextClass = window.AudioContext || (window as any).webkitAudioContext;
this.ctx = new AudioContextClass();
this.inputGain = this.ctx.createGain();
this.inputDestination = this.ctx.createMediaStreamDestination();
this.analyser = this.ctx.createAnalyser();
this.analyser.fftSize = 256;
this.silentGain = this.ctx.createGain();
this.silentGain.gain.value = 0;
this.inputGain.connect(this.inputDestination);
this.inputGain.connect(this.analyser);
this.inputGain.connect(this.silentGain);
this.silentGain.connect(this.ctx.destination);
this.inputGain.gain.setValueAtTime(1, this.ctx.currentTime);
this.ctx.onstatechange = () => {
console.log(`[AudioManager] Context state: ${this.ctx?.state}`);
if (this.ctx?.state === 'running') {
this.notifyResumed();
}
};
this.isInitialized = true;
}
onResumed(cb: () => void) {
this.listeners.add(cb);
return () => this.listeners.delete(cb);
}
private notifyResumed() {
this.listeners.forEach(cb => cb());
}
async resumeContext() {
if (!this.ctx) this.initContext();
if (this.ctx && this.ctx.state === 'suspended') {
try {
await this.ctx.resume();
console.log('[AudioManager] AudioContext resumed.');
} catch (err) {
console.error('[AudioManager] Failed to resume context:', err);
}
}
}
async setInputDevice(deviceId: string) {
if (!this.isInitialized) this.initContext();
// Skip if already set and stream is active
if (this.currentInputDeviceId === deviceId && this.currentStream?.active) {
return this.currentStream;
}
try {
if (this.currentStream) {
this.currentStream.getTracks().forEach(t => t.stop());
}
const constraints = {
audio: {
deviceId: deviceId === 'default' ? undefined : { exact: deviceId },
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true
}
};
this.currentStream = await navigator.mediaDevices.getUserMedia(constraints);
this.currentInputDeviceId = deviceId;
if (this.ctx && this.inputGain) {
if (this.inputSource) {
this.inputSource.disconnect();
}
this.inputSource = this.ctx.createMediaStreamSource(this.currentStream);
this.inputSource.connect(this.inputGain);
}
return this.currentStream;
} catch (err) {
console.error('[AudioManager] Failed to set input device:', err);
throw err;
}
}
setInputVolume(volume: number) {
if (!this.isInitialized) this.initContext();
if (this.inputGain && this.ctx) {
const gainValue = volume / 100;
this.inputGain.gain.setTargetAtTime(gainValue, this.ctx.currentTime, 0.1);
}
}
/**
* CRITICAL: Always returns a CLONE of the destination track.
* This prevents LiveKit's cleanup from killing the main singleton track
* when switching rooms.
*/
getFreshTrack(): MediaStreamTrack | null {
if (!this.isInitialized) this.initContext();
const track = this.inputDestination!.stream.getAudioTracks()[0];
if (!track) return null;
return track.clone();
}
getAnalyserNode(): AnalyserNode {
if (!this.isInitialized) this.initContext();
return this.analyser!;
}
getContext(): AudioContext | null {
return this.ctx;
}
}
packages/web/src/components/layout/AppLayout.tsx
+28
View File
@@ -23,9 +23,37 @@ import { useServerStore } from '../../stores/serverStore';
import { useChatStore } from '../../stores/chatStore';
import { useUIStore } from '../../stores/uiStore';
import { useVoiceStore } from '../../stores/voiceStore';
import { AudioManager } from '../../audio/AudioManager';
export function AppLayout() {
const { serverId, channelId, inviteCode } = useParams<{ serverId?: string; channelId?: string; inviteCode?: string }>();
// Global interaction handler to resume AudioContext
useEffect(() => {
const resume = () => {
AudioManager.getInstance().resumeContext().then(() => {
// Wake up all audio/video elements that might be blocked by Autoplay
document.querySelectorAll('audio, video').forEach(el => {
(el as HTMLMediaElement).play().catch(() => {
// Silently fail if still blocked or no source
});
});
window.removeEventListener('click', resume);
window.removeEventListener('keydown', resume);
window.removeEventListener('touchstart', resume);
});
};
window.addEventListener('click', resume);
window.addEventListener('keydown', resume);
window.addEventListener('touchstart', resume);
return () => {
window.removeEventListener('click', resume);
window.removeEventListener('keydown', resume);
window.removeEventListener('touchstart', resume);
};
}, []);
const { user, isLoading } = useAuth();
const setCurrentServer = useServerStore((s) => s.setCurrentServer);
const loadServerDetail = useServerStore((s) => s.loadServerDetail);
packages/web/src/components/layout/ChannelSidebar.tsx
+37 -55
View File
@@ -10,6 +10,7 @@ import { useVoiceStore } from '../../stores/voiceStore';
import { Avatar } from '../ui/Avatar';
import { wsSend } from '../../hooks/useWebSocket';
import { getActiveRoom } from '../../hooks/useLiveKit';
import { AudioManager } from '../../audio/AudioManager';
export function ChannelSidebar() {
const servers = useServerStore((s) => s.servers);
@@ -31,14 +32,6 @@ export function ChannelSidebar() {
const navigate = useNavigate();
const handleMicToggle = async () => {
const room = getActiveRoom();
if (room) {
try {
await room.localParticipant.setMicrophoneEnabled(isMuted);
} catch (err) {
console.error('[ChannelSidebar] Failed to toggle mic:', err);
}
}
toggleMic();
// Broadcast mute status via WebSocket so non-joined users can see it
const willBeMuted = !isMuted;
@@ -57,15 +50,6 @@ export function ChannelSidebar() {
wsSend({ type: 'voice_status', isMuted: willBeMuted, isDeafened: willDeafen });
if (room) {
try {
if (willDeafen) {
await room.localParticipant.setMicrophoneEnabled(false);
room.remoteParticipants.forEach((p) => p.setVolume(0));
} else {
const outputVolume = useVoiceStore.getState().outputVolume;
const scaled = outputVolume / 100;
room.remoteParticipants.forEach((p) => p.setVolume(scaled));
await room.localParticipant.setMicrophoneEnabled(true);
}
// Broadcast deafen state to other participants via LiveKit data channel
const encoder = new TextEncoder();
room.localParticipant.publishData(
@@ -390,10 +374,14 @@ function UserAreaPanel({
const [openPanel, setOpenPanel] = useState<'input' | 'output' | null>(null);
const [inputDevices, setInputDevices] = useState<MediaDeviceInfo[]>([]);
const [outputDevices, setOutputDevices] = useState<MediaDeviceInfo[]>([]);
const [selectedInput, setSelectedInput] = useState<string>('default');
const [selectedOutput, setSelectedOutput] = useState<string>('default');
const inputDeviceId = useVoiceStore((s) => s.inputDeviceId);
const outputDeviceId = useVoiceStore((s) => s.outputDeviceId);
const setInputDevice = useVoiceStore((s) => s.setInputDevice);
const setOutputDevice = useVoiceStore((s) => s.setOutputDevice);
const [selectedInputLabel, setSelectedInputLabel] = useState<string>('Default');
const [selectedOutputLabel, setSelectedOutputLabel] = useState<string>('Default');
const inputVolume = useVoiceStore((s) => s.inputVolume);
const storeSetInputVolume = useVoiceStore((s) => s.setInputVolume);
const outputVolume = useVoiceStore((s) => s.outputVolume);
@@ -408,14 +396,24 @@ function UserAreaPanel({
const loadDevices = useCallback(async () => {
try {
// Need to request permission first to get labels
if (!AudioManager.getInstance().getContext()) {
await navigator.mediaDevices.getUserMedia({ audio: true }).then(s => s.getTracks().forEach(t => t.stop()));
}
const devices = await navigator.mediaDevices.enumerateDevices();
setInputDevices(devices.filter(d => d.kind === 'audioinput'));
setOutputDevices(devices.filter(d => d.kind === 'audiooutput'));
const inputs = devices.filter(d => d.kind === 'audioinput');
const outputs = devices.filter(d => d.kind === 'audiooutput');
setInputDevices(inputs);
setOutputDevices(outputs);
const currentInput = inputs.find(d => d.deviceId === inputDeviceId);
if (currentInput) setSelectedInputLabel(currentInput.label || 'Default');
const currentOutput = outputs.find(d => d.deviceId === outputDeviceId);
if (currentOutput) setSelectedOutputLabel(currentOutput.label || 'Default');
} catch {
// permission denied
}
}, []);
}, [inputDeviceId, outputDeviceId]);
// Start mic level monitoring when input panel opens
useEffect(() => {
@@ -425,17 +423,14 @@ function UserAreaPanel({
setMicLevel(0);
return;
}
let stream: MediaStream | null = null;
let ctx: AudioContext | null = null;
const start = async () => {
try {
stream = await navigator.mediaDevices.getUserMedia({ audio: { deviceId: selectedInput !== 'default' ? selectedInput : undefined } });
ctx = new AudioContext();
const source = ctx.createMediaStreamSource(stream);
const analyser = ctx.createAnalyser();
await AudioManager.getInstance().resumeContext();
const analyser = AudioManager.getInstance().getAnalyserNode();
analyser.fftSize = 256;
source.connect(analyser);
analyserRef.current = analyser;
const data = new Uint8Array(analyser.frequencyBinCount);
const tick = () => {
if (!analyserRef.current) return;
@@ -451,10 +446,8 @@ function UserAreaPanel({
return () => {
if (animFrameRef.current) cancelAnimationFrame(animFrameRef.current);
analyserRef.current = null;
stream?.getTracks().forEach(t => t.stop());
ctx?.close();
};
}, [openPanel, selectedInput]);
}, [openPanel]);
useEffect(() => {
const handleClick = (e: MouseEvent) => {
@@ -476,19 +469,19 @@ function UserAreaPanel({
setOpenPanel(panel);
setShowInputDeviceList(false);
setShowOutputDeviceList(false);
// Explicitly resume on interaction
AudioManager.getInstance().resumeContext();
}
};
const selectInput = (device: MediaDeviceInfo) => {
setSelectedInput(device.deviceId);
setInputDevice(device.deviceId);
setSelectedInputLabel(device.label || 'Default');
setShowInputDeviceList(false);
const room = getActiveRoom();
if (room) room.switchActiveDevice('audioinput', device.deviceId).catch(() => {});
};
const selectOutput = (device: MediaDeviceInfo) => {
setSelectedOutput(device.deviceId);
setOutputDevice(device.deviceId);
setSelectedOutputLabel(device.label || 'Default');
setShowOutputDeviceList(false);
const room = getActiveRoom();
@@ -525,15 +518,15 @@ function UserAreaPanel({
key={d.deviceId}
onClick={() => selectInput(d)}
className={`w-full px-3 py-2 text-left text-[13px] hover:bg-discord-modifier-hover transition-colors flex items-center gap-2 ${
selectedInput === d.deviceId ? 'text-discord-text-primary' : 'text-discord-text-secondary'
inputDeviceId === d.deviceId ? 'text-discord-text-primary' : 'text-discord-text-secondary'
}`}
>
{selectedInput === d.deviceId && (
{inputDeviceId === d.deviceId && (
<svg width="16" height="16" viewBox="0 0 24 24" fill="currentColor" className="text-discord-blurple flex-shrink-0">
<path d="M9 16.17L4.83 12l-1.42 1.41L9 19 21 7l-1.41-1.41z" />
</svg>
)}
<span className={selectedInput === d.deviceId ? '' : 'pl-6'}>{d.label || 'Default'}</span>
<span className={inputDeviceId === d.deviceId ? '' : 'pl-6'}>{d.label || 'Default'}</span>
</button>
))}
</div>
@@ -553,19 +546,8 @@ function UserAreaPanel({
onChange={(e) => {
const vol = Number(e.target.value);
storeSetInputVolume(vol);
const room = getActiveRoom();
if (room) {
const { isMuted: manuallyMuted, isDeafened: manuallyDeafened } = useVoiceStore.getState();
// If user is manually muted, hardware should stay off regardless of volume.
// If user is NOT manually muted and volume is 0, we can keep hardware ON
// (Web Audio handles silence) or turn it OFF for battery/privacy.
// Discord keeps it ON (green ring) but silent. We'll follow that.
if (!manuallyMuted && !manuallyDeafened && !room.localParticipant.isMicrophoneEnabled && vol > 0) {
room.localParticipant.setMicrophoneEnabled(true).catch(() => {});
}
}
}} className="w-full h-1.5 rounded-full appearance-none cursor-pointer accent-discord-blurple bg-discord-bg-tertiary [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:w-3 [&::-webkit-slider-thumb]:h-3 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-white [&::-webkit-slider-thumb]:shadow-md"
}}
className="w-full h-1.5 rounded-full appearance-none cursor-pointer accent-discord-blurple bg-discord-bg-tertiary [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:w-3 [&::-webkit-slider-thumb]:h-3 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-white [&::-webkit-slider-thumb]:shadow-md"
style={{
background: `linear-gradient(to right, #5865f2 0%, #5865f2 ${inputVolume / 2}%, #4e5058 ${inputVolume / 2}%, #4e5058 100%)`,
}}
@@ -622,15 +604,15 @@ function UserAreaPanel({
key={d.deviceId}
onClick={() => selectOutput(d)}
className={`w-full px-3 py-2 text-left text-[13px] hover:bg-discord-modifier-hover transition-colors flex items-center gap-2 ${
selectedOutput === d.deviceId ? 'text-discord-text-primary' : 'text-discord-text-secondary'
outputDeviceId === d.deviceId ? 'text-discord-text-primary' : 'text-discord-text-secondary'
}`}
>
{selectedOutput === d.deviceId && (
{outputDeviceId === d.deviceId && (
<svg width="16" height="16" viewBox="0 0 24 24" fill="currentColor" className="text-discord-blurple flex-shrink-0">
<path d="M9 16.17L4.83 12l-1.42 1.41L9 19 21 7l-1.41-1.41z" />
</svg>
)}
<span className={selectedOutput === d.deviceId ? '' : 'pl-6'}>{d.label || 'Default'}</span>
<span className={outputDeviceId === d.deviceId ? '' : 'pl-6'}>{d.label || 'Default'}</span>
</button>
))}
</div>
packages/web/src/components/voice/VoiceControlBar.tsx
-16
View File
@@ -36,14 +36,6 @@ export function VoiceControlBar() {
const [qualityOpen, setQualityOpen] = useState(false);
const handleMute = React.useCallback(async () => {
const room = getActiveRoom();
if (room) {
try {
await room.localParticipant.setMicrophoneEnabled(isMuted);
} catch (err) {
console.error('[VoiceControlBar] Failed to toggle mic:', err);
}
}
toggleMic();
// Broadcast via WebSocket so sidebar shows status without joining
wsSend({ type: 'voice_status', isMuted: !isMuted, isDeafened });
@@ -60,14 +52,6 @@ export function VoiceControlBar() {
wsSend({ type: 'voice_status', isMuted: willDeafen, isDeafened: willDeafen });
if (room) {
try {
if (willDeafen) {
await room.localParticipant.setMicrophoneEnabled(false);
room.remoteParticipants.forEach((p) => p.setVolume(0));
} else {
const outputVolume = useVoiceStore.getState().outputVolume;
room.remoteParticipants.forEach((p) => p.setVolume(outputVolume / 100));
await room.localParticipant.setMicrophoneEnabled(true);
}
// Broadcast deafen state via LiveKit data channel for in-room users
const encoder = new TextEncoder();
room.localParticipant.publishData(
packages/web/src/components/voice/VoiceUser.tsx
+121 -113
View File
@@ -1,7 +1,7 @@
import React, { useRef, useEffect, useState, useCallback } from 'react';
import { Avatar } from '../ui/Avatar';
import { useVoiceStore } from '../../stores/voiceStore';
import { getSharedAudioCtx } from '../../hooks/useLiveKit';
import { AudioManager } from '../../audio/AudioManager';
import type { ParticipantInfo } from '../../hooks/useLiveKit';
interface VoiceUserProps {
@@ -12,100 +12,132 @@ interface VoiceUserProps {
export function VoiceUser({ participant, large }: VoiceUserProps) {
const videoRef = useRef<HTMLVideoElement>(null);
const audioRef = useRef<HTMLAudioElement>(null);
const isDeafened = useVoiceStore((s) => s.isDeafened);
const outputVolume = useVoiceStore((s) => s.outputVolume);
const participantVolumes = useVoiceStore((s) => s.participantVolumes);
const [, forceUpdate] = useState(0);
const perUserVolume = participantVolumes.get(participant.userId) ?? 100;
const isLocal = participant.isLocal;
// Web Audio for volume boost (> 100%)
const gainNodeRef = useRef<GainNode | null>(null);
const sourceNodeRef = useRef<MediaStreamAudioSourceNode | null>(null);
// --- AUDIO PIPELINE: NATIVE FIRST ---
// Refs for the optional boost pipeline
const boostGainRef = useRef<GainNode | null>(null);
const boostSourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
// 1. Basic Track Attachment (The Rock-Solid Foundation)
useEffect(() => {
const audioEl = audioRef.current;
if (isLocal || !audioEl || !participant.audioTrack) return;
// Direct attachment.
const stream = new MediaStream([participant.audioTrack]);
// Only update if changed to prevent interruptions
if ((audioEl.srcObject as MediaStream)?.id !== stream.id) {
audioEl.srcObject = stream;
// Aggressive play attempt for Chrome
const tryPlay = async () => {
try {
await audioEl.play();
} catch (err) {
console.warn("[Audio] Autoplay blocked, retrying...", err);
// If blocked, we rely on the global interaction listener to resume context,
// but we can also retry play() on the element itself on next click.
}
};
tryPlay();
}
}, [participant.audioTrack, isLocal]);
// 2. Volume Management (Hybrid)
useEffect(() => {
const audioEl = audioRef.current;
if (isLocal || !audioEl || !participant.audioTrack) return;
const globalScale = outputVolume / 100;
const userScale = perUserVolume / 100;
const finalVolume = globalScale * userScale;
if (isDeafened) {
audioEl.muted = true;
return;
}
// Logic:
// If we are boosting (>100%) AND context is running, use Web Audio.
// Otherwise, stick to the native element for maximum reliability.
const audioManager = AudioManager.getInstance();
const ctx = audioManager.getContext();
const isBoosting = finalVolume > 1.0;
const isContextReady = ctx && ctx.state === 'running';
if (isBoosting && isContextReady) {
// --- BOOST MODE (>100%) ---
// Setup pipeline if missing
if (!boostGainRef.current && ctx) {
const gain = ctx.createGain();
const source = ctx.createMediaStreamSource(new MediaStream([participant.audioTrack]));
source.connect(gain);
gain.connect(ctx.destination);
boostGainRef.current = gain;
boostSourceRef.current = source;
}
// Apply boosted gain
if (boostGainRef.current && ctx) {
boostGainRef.current.gain.setTargetAtTime(finalVolume, ctx.currentTime, 0.01);
}
// MUTE the element so we don't double audio
audioEl.muted = true;
} else {
// --- STANDARD MODE (0% - 100%) ---
// Clean up boost pipeline if it exists
if (boostSourceRef.current) {
boostSourceRef.current.disconnect();
boostSourceRef.current = null;
boostGainRef.current = null;
}
// Use the element
audioEl.muted = false;
audioEl.volume = Math.min(finalVolume, 1.0);
// Ensure it's playing (in case it was paused/blocked earlier)
if (audioEl.paused) {
audioEl.play().catch(() => {});
}
}
}, [outputVolume, perUserVolume, isDeafened, isLocal, participant.audioTrack]);
// --- VIDEO & UI ---
// Determine active video track
const liveScreen = participant.isScreenSharing && participant.screenTrack?.readyState === 'live' ? participant.screenTrack : null;
const liveCamera = participant.isCameraOn && participant.videoTrack?.readyState === 'live' ? participant.videoTrack : null;
const activeVideoTrack = liveScreen ?? liveCamera;
const hasVideo = activeVideoTrack !== null;
const isScreenShare = liveScreen !== null;
// 1. STANDARD AUDIO PLAYBACK (Reliability Layer)
// Force re-render when tracks end/mute
useEffect(() => {
const audioEl = audioRef.current;
if (isLocal || !audioEl || !participant.audioTrack) return;
const tracks = [participant.videoTrack, participant.screenTrack].filter((t): t is MediaStreamTrack => t !== null);
if (tracks.length === 0) return;
const onEnded = () => forceUpdate((n) => n + 1);
tracks.forEach((t) => t.addEventListener('ended', onEnded));
return () => tracks.forEach((t) => t.removeEventListener('ended', onEnded));
}, [participant.videoTrack, participant.screenTrack]);
const stream = new MediaStream([participant.audioTrack]);
if ((audioEl.srcObject as MediaStream)?.id !== stream.id) {
audioEl.srcObject = stream;
// Critical for Chrome: Explicitly call play()
audioEl.play().catch((err) => console.warn('[Audio] Auto-play blocked:', err));
}
}, [participant.audioTrack, isLocal]);
// 2. VOLUME & BOOST CONTROL
useEffect(() => {
const audioEl = audioRef.current;
if (isLocal || !audioEl || !participant.audioTrack) return;
// Calculate total requested volume (0.0 to 2.0+)
const combined = (perUserVolume / 100) * (outputVolume / 100);
if (isDeafened) {
audioEl.muted = true;
if (gainNodeRef.current) gainNodeRef.current.gain.value = 0;
return;
}
// Logic:
// 0% - 100%: Use standard <audio> volume. Disconnect Web Audio to prevent doubling.
// > 100%: Set <audio> to 100%, connect Web Audio for the EXTRA boost.
// Standard Path (Always Active unless >100% needs to take over completely, but doubling is risk.
// SAFE APPROACH: Use <audio> for everything up to 100%.
// If > 100%, keep <audio> at 100% and add Web Audio *parallel*? No, that causes phasing.
// CORRECT APPROACH:
// If <= 100%: Element Volume = combined. Web Audio = Disconnected.
// If > 100%: Element Volume = 0 (Muted). Web Audio = connected & combined.
const ctx = getSharedAudioCtx();
const useWebAudio = combined > 1.0 && ctx && ctx.state === 'running';
if (useWebAudio) {
// --- BOOST MODE (>100%) ---
// Mute standard element to prevent double audio
audioEl.muted = true;
// Setup/Connect Web Audio
if (!gainNodeRef.current) {
gainNodeRef.current = ctx.createGain();
gainNodeRef.current.connect(ctx.destination);
}
if (!sourceNodeRef.current) {
sourceNodeRef.current = ctx.createMediaStreamSource(new MediaStream([participant.audioTrack]));
sourceNodeRef.current.connect(gainNodeRef.current);
}
// Apply full gain (e.g., 1.5, 2.0)
gainNodeRef.current.gain.setTargetAtTime(combined, ctx.currentTime, 0.01);
} else {
// --- STANDARD MODE (0-100%) ---
// Cleanup Web Audio to prevent doubling/leaking
if (sourceNodeRef.current) {
sourceNodeRef.current.disconnect();
sourceNodeRef.current = null;
}
// Use standard element
audioEl.muted = false;
audioEl.volume = Math.min(combined, 1.0);
}
}, [isDeafened, outputVolume, perUserVolume, isLocal, participant.audioTrack]);
// Video Handling
// Attach Video
useEffect(() => {
const videoEl = videoRef.current;
if (!videoEl) return;
@@ -116,32 +148,15 @@ export function VoiceUser({ participant, large }: VoiceUserProps) {
}
}, [activeVideoTrack]);
// Cleanup Listeners
useEffect(() => {
const tracks = [participant.videoTrack, participant.screenTrack].filter((t): t is MediaStreamTrack => t !== null);
if (tracks.length === 0) return;
const onEnded = () => forceUpdate((n) => n + 1);
tracks.forEach((t) => t.addEventListener('ended', onEnded));
return () => tracks.forEach((t) => t.removeEventListener('ended', onEnded));
}, [participant.videoTrack, participant.screenTrack]);
// Interaction (Resume Context)
const handleInteraction = useCallback(() => {
const ctx = getSharedAudioCtx();
if (ctx && ctx.state === 'suspended') {
ctx.resume().catch(console.error);
}
}, []);
const setParticipantVolume = useVoiceStore((s) => s.setParticipantVolume);
// Context Menu
const [volumeMenu, setVolumeMenu] = useState<{ x: number; y: number } | null>(null);
const setParticipantVolume = useVoiceStore((s) => s.setParticipantVolume);
const handleContextMenu = useCallback((e: React.MouseEvent) => {
if (isLocal) return;
e.preventDefault();
handleInteraction();
setVolumeMenu({ x: e.clientX, y: e.clientY });
}, [isLocal, handleInteraction]);
}, [isLocal]);
useEffect(() => {
if (!volumeMenu) return;
@@ -152,7 +167,6 @@ export function VoiceUser({ participant, large }: VoiceUserProps) {
return (
<div
onClick={handleInteraction}
className={`relative bg-[#111214] rounded-xl overflow-hidden flex items-center justify-center group transition-all duration-200 ${
participant.isSpeaking
? 'ring-[3px] ring-discord-green shadow-[0_0_12px_rgba(35,165,90,0.25)]'
@@ -160,7 +174,11 @@ export function VoiceUser({ participant, large }: VoiceUserProps) {
} ${large ? 'h-full w-full' : 'h-full aspect-video'}`}
onContextMenu={handleContextMenu}
>
{/* Audio Element: Primary playback device */}
{/*
Native Audio Element
- AutoPlay is critical
- PlaysInline is critical for mobile
*/}
{!isLocal && <audio ref={audioRef} autoPlay playsInline />}
{hasVideo ? (
@@ -227,13 +245,7 @@ export function VoiceUser({ participant, large }: VoiceUserProps) {
User Volume
</div>
<div className="flex items-center gap-2">
<svg
width="16"
height="16"
viewBox="0 0 24 24"
fill="currentColor"
className="text-discord-text-muted flex-shrink-0"
>
<svg width="16" height="16" viewBox="0 0 24 24" fill="currentColor" className="text-discord-text-muted flex-shrink-0">
<path d="M3 9v6h4l5 5V4L7 9H3z" />
</svg>
<input
@@ -241,14 +253,10 @@ export function VoiceUser({ participant, large }: VoiceUserProps) {
min="0"
max="200"
value={perUserVolume}
onChange={(e) =>
setParticipantVolume(participant.userId, parseInt(e.target.value))
}
onChange={(e) => setParticipantVolume(participant.userId, parseInt(e.target.value))}
className="flex-1 accent-discord-blurple h-1"
/>
<span className="text-xs text-discord-text-secondary min-w-[32px] text-right">
{perUserVolume}%
</span>
<span className="text-xs text-discord-text-secondary min-w-[32px] text-right">{perUserVolume}%</span>
</div>
</div>
)}
packages/web/src/hooks/useLiveKit.ts
+70 -91
View File
@@ -9,9 +9,11 @@ import {
VideoPresets,
VideoPreset,
LocalAudioTrack,
LocalTrackPublication,
} from 'livekit-client';
import { api } from '../api/client';
import { useVoiceStore } from '../stores/voiceStore';
import { AudioManager } from '../audio/AudioManager';
/**
* OPENCORD NATIVE OVERDRIVE PIPELINE v32
@@ -29,31 +31,6 @@ const QUALITY_MAP: Record<string, VideoPreset> = {
const AUTO_PRESET = QUALITY_MAP['720p60']!;
let _activeRoom: Room | null = null;
let _sharedAudioCtx: AudioContext | null = null;
export function getSharedAudioCtx() {
if (typeof window === 'undefined') return null;
if (!_sharedAudioCtx) {
_sharedAudioCtx = new (window.AudioContext || (window as any).webkitAudioContext)();
}
return _sharedAudioCtx;
}
// Global gesture resumer
if (typeof window !== 'undefined') {
const resume = () => {
const ctx = getSharedAudioCtx();
if (ctx && ctx.state === 'suspended') {
ctx.resume().then(() => {
console.log('[Audio] Shared context resumed via interaction');
window.removeEventListener('click', resume);
window.removeEventListener('keydown', resume);
});
}
};
window.addEventListener('click', resume);
window.addEventListener('keydown', resume);
}
export function getActiveRoom(): Room | null {
return _activeRoom;
@@ -123,11 +100,7 @@ export function useLiveKit() {
const videoQuality = useVoiceStore((s) => s.videoQuality);
const voiceUserStates = useVoiceStore((s) => s.voiceUserStates);
const inputVolume = useVoiceStore((s) => s.inputVolume);
// Web Audio for Local Input Gain
const localGainNodeRef = useRef<GainNode | null>(null);
const localSourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
const localDestRef = useRef<MediaStreamAudioDestinationNode | null>(null);
const inputDeviceId = useVoiceStore((s) => s.inputDeviceId);
const updateParticipants = useCallback(() => {
const r = roomRef.current;
@@ -168,14 +141,6 @@ export function useLiveKit() {
setParticipants(allParticipants);
}, []);
// Sync Input Gain value
useEffect(() => {
if (localGainNodeRef.current) {
const ctx = getSharedAudioCtx();
localGainNodeRef.current.gain.setTargetAtTime(inputVolume / 100, ctx?.currentTime || 0, 0.01);
}
}, [inputVolume]);
const handleDataReceived = useCallback((payload: Uint8Array, participant?: RemoteParticipant) => {
try {
const text = new TextDecoder().decode(payload);
@@ -188,39 +153,66 @@ export function useLiveKit() {
} catch { }
}, [updateParticipants]);
const setupLocalGainPipeline = useCallback(async (room: Room, audioTrack: LocalAudioTrack) => {
// Handle Input Device & Mute Logic via AudioManager
useEffect(() => {
const r = roomRef.current;
if (!r || !isConnected) return;
const syncMic = async () => {
try {
const ctx = getSharedAudioCtx();
if (!ctx || !audioTrack.mediaStreamTrack) return;
const audioManager = AudioManager.getInstance();
if (!localGainNodeRef.current) {
localGainNodeRef.current = ctx.createGain();
localDestRef.current = ctx.createMediaStreamDestination();
localGainNodeRef.current.connect(localDestRef.current);
// If muted or deafened, unpublish mic
if (isMuted || isDeafened) {
const pub = r.localParticipant.getTrackPublications().find(p => p.source === Track.Source.Microphone);
if (pub) {
await r.localParticipant.unpublishTrack(pub.track as LocalAudioTrack);
}
return;
}
if (localSourceRef.current) localSourceRef.current.disconnect();
localSourceRef.current = ctx.createMediaStreamSource(new MediaStream([audioTrack.mediaStreamTrack]));
localSourceRef.current.connect(localGainNodeRef.current!);
// Ensure device is set and volume is sync'd
await audioManager.setInputDevice(inputDeviceId);
audioManager.setInputVolume(inputVolume);
// Set gain from store
localGainNodeRef.current!.gain.value = useVoiceStore.getState().inputVolume / 100;
// Check if already published
const existingPub = r.localParticipant.getTrackPublications().find(p => p.source === Track.Source.Microphone);
const processedTrack = localDestRef.current!.stream.getAudioTracks()[0];
const engine = (room as any).engine;
const pc = engine?.pcManager?.publisher?.pc || engine?.publisher?.pc || engine?.pc;
if (pc && processedTrack) {
const senders = (pc as RTCPeerConnection).getSenders();
const sender = senders.find(s => s.track?.id === audioTrack.mediaStreamTrack.id);
if (sender) {
console.log('[LiveKit] Swapping raw mic for gain-processed track');
await sender.replaceTrack(processedTrack);
if (existingPub && existingPub.track) {
// If track is alive, we are good.
if (existingPub.track.mediaStreamTrack?.readyState === 'live') {
return;
}
// If track died, unpublish so we can republish
await r.localParticipant.unpublishTrack(existingPub.track as LocalAudioTrack);
}
// Get a FRESH track (clone) for this specific publication
const audioTrack = audioManager.getFreshTrack();
if (!audioTrack) return;
console.log('[LiveKit] Publishing fresh microphone track');
await r.localParticipant.publishTrack(audioTrack, {
name: 'microphone',
source: Track.Source.Microphone,
});
} catch (err) {
console.error('[LiveKit] Local gain setup failed:', err);
console.error('[LiveKit] Failed to sync mic state:', err);
}
}, []);
};
syncMic();
// Re-sync when AudioManager resumes
const unsubscribe = AudioManager.getInstance().onResumed(() => {
syncMic();
});
return () => {
unsubscribe();
};
}, [isMuted, isDeafened, inputDeviceId, inputVolume, isConnected, room]);
const connect = useCallback(async (channelId: string) => {
if (connectedChannelRef.current === channelId && roomRef.current) return;
@@ -248,12 +240,7 @@ export function useLiveKit() {
newRoom.on(RoomEvent.ParticipantDisconnected, guardedUpdate);
newRoom.on(RoomEvent.TrackSubscribed, guardedUpdate);
newRoom.on(RoomEvent.TrackUnsubscribed, guardedUpdate);
newRoom.on(RoomEvent.LocalTrackPublished, (pub) => {
guardedUpdate();
if (pub.source === Track.Source.Microphone && pub.track instanceof LocalAudioTrack) {
setupLocalGainPipeline(newRoom, pub.track);
}
});
newRoom.on(RoomEvent.LocalTrackPublished, guardedUpdate);
newRoom.on(RoomEvent.LocalTrackUnpublished, guardedUpdate);
newRoom.on(RoomEvent.TrackMuted, guardedUpdate);
newRoom.on(RoomEvent.TrackUnmuted, guardedUpdate);
@@ -280,21 +267,20 @@ export function useLiveKit() {
updateParticipants();
// Initial mute state check
const { isMuted: wasMuted, isDeafened: wasDeafened } = useVoiceStore.getState();
useVoiceStore.setState({ isCameraOn: false, isScreenSharing: false });
if (!wasMuted && !wasDeafened) {
await newRoom.localParticipant.setMicrophoneEnabled(true);
} else {
await newRoom.localParticipant.setMicrophoneEnabled(false);
// Mic handling is now done by useEffect
if (wasDeafened) {
newRoom.remoteParticipants.forEach((p) => p.setVolume(0));
}
}
updateParticipants();
} catch (err) { if (gen === _connectGeneration) setConnectionError('Failed to connect'); }
finally { if (gen === _connectGeneration) setIsConnecting(false); }
}, [updateParticipants, handleDataReceived, setupLocalGainPipeline]);
}, [updateParticipants, handleDataReceived]);
const connectDm = useCallback(async (dmChannelId: string) => {
const gen = ++_connectGeneration;
@@ -310,12 +296,7 @@ export function useLiveKit() {
newRoom.on(RoomEvent.ParticipantDisconnected, guardedUpdate);
newRoom.on(RoomEvent.TrackSubscribed, guardedUpdate);
newRoom.on(RoomEvent.TrackUnsubscribed, guardedUpdate);
newRoom.on(RoomEvent.LocalTrackPublished, (pub) => {
guardedUpdate();
if (pub.source === Track.Source.Microphone && pub.track instanceof LocalAudioTrack) {
setupLocalGainPipeline(newRoom, pub.track);
}
});
newRoom.on(RoomEvent.LocalTrackPublished, guardedUpdate);
newRoom.on(RoomEvent.LocalTrackUnpublished, guardedUpdate);
newRoom.on(RoomEvent.TrackMuted, guardedUpdate);
newRoom.on(RoomEvent.TrackUnmuted, guardedUpdate);
@@ -334,18 +315,16 @@ export function useLiveKit() {
updateParticipants();
const { isMuted: wasMuted, isDeafened: wasDeafened } = useVoiceStore.getState();
useVoiceStore.setState({ isCameraOn: false, isScreenSharing: false });
if (!wasMuted && !wasDeafened) {
await newRoom.localParticipant.setMicrophoneEnabled(true);
} else {
await newRoom.localParticipant.setMicrophoneEnabled(false);
// Mic handling is now done by useEffect
if (wasDeafened) {
newRoom.remoteParticipants.forEach((p) => p.setVolume(0));
}
}
updateParticipants();
} catch (err) { if (gen === _connectGeneration) setConnectionError('Failed to connect'); }
finally { if (gen === _connectGeneration) setIsConnecting(false); }
}, [updateParticipants, handleDataReceived, setupLocalGainPipeline]);
}, [updateParticipants, handleDataReceived]);
const disconnect = useCallback(async () => {
_connectGeneration++;
@@ -353,11 +332,11 @@ export function useLiveKit() {
}, []);
const toggleMic = useCallback(async () => {
if (roomRef.current) {
await roomRef.current.localParticipant.setMicrophoneEnabled(!isMuted);
updateParticipants();
}
}, [isMuted, updateParticipants]);
// This is now purely a UI helper, actual toggling logic is in the store and useEffect
// But we might want to manually trigger resume here just in case
await AudioManager.getInstance().resumeContext();
useVoiceStore.getState().toggleMic();
}, []);
const toggleCamera = useCallback(async () => {
if (roomRef.current) {
@@ -453,5 +432,5 @@ export function useLiveKit() {
return () => { _connectGeneration++; if (roomRef.current) { roomRef.current.disconnect(); roomRef.current = null; _activeRoom = null; } };
}, []);
return { room, participants, isConnected, isConnecting, connectionError, connect, connectDm, disconnect, toggleMic, toggleCamera, toggleScreenShare, getSharedAudioCtx };
return { room, participants, isConnected, isConnecting, connectionError, connect, connectDm, disconnect, toggleMic, toggleCamera, toggleScreenShare };
}
packages/web/src/stores/voiceStore.ts
+22 -1
View File
@@ -1,6 +1,7 @@
import { create } from 'zustand';
import { persist, createJSONStorage } from 'zustand/middleware';
import type { ParticipantInfo } from '../hooks/useLiveKit';
import { AudioManager } from '../audio/AudioManager';
interface VoiceState {
voiceUsers: Map<string, string[]>; // channelId → userIds
@@ -14,6 +15,8 @@ interface VoiceState {
isLiveKitConnected: boolean;
inputVolume: number; // 0-200 (100 = default)
outputVolume: number; // 0-200 (100 = default)
inputDeviceId: string;
outputDeviceId: string;
focusedParticipantId: string | null;
videoQuality: '1080p' | '1080p60' | '720p' | '720p60' | '540p' | '360p';
// Per-participant volume (userId → 0-200, 100 = default)
@@ -36,6 +39,8 @@ interface VoiceState {
setIsLiveKitConnected: (connected: boolean) => void;
setInputVolume: (volume: number) => void;
setOutputVolume: (volume: number) => void;
setInputDevice: (deviceId: string) => Promise<void>;
setOutputDevice: (deviceId: string) => void;
toggleMic: () => void;
toggleCamera: () => void;
toggleScreenShare: () => void;
@@ -70,6 +75,8 @@ export const useVoiceStore = create<VoiceState>()(
isLiveKitConnected: false,
inputVolume: 100,
outputVolume: 100,
inputDeviceId: 'default',
outputDeviceId: 'default',
focusedParticipantId: null,
videoQuality: '720p60',
participantVolumes: new Map(),
@@ -124,9 +131,19 @@ export const useVoiceStore = create<VoiceState>()(
setConnectionError: (error) => set({ connectionError: error }),
setIsLiveKitConnected: (connected) => set({ isLiveKitConnected: connected }),
setInputVolume: (volume) => set({ inputVolume: volume }),
setInputVolume: (volume) => {
set({ inputVolume: volume });
AudioManager.getInstance().setInputVolume(volume);
},
setOutputVolume: (volume) => set({ outputVolume: volume }),
setInputDevice: async (deviceId) => {
set({ inputDeviceId: deviceId });
await AudioManager.getInstance().setInputDevice(deviceId);
},
setOutputDevice: (deviceId) => set({ outputDeviceId: deviceId }),
toggleMic: () => set((state) => ({ isMuted: !state.isMuted })),
toggleDeafen: () => set((state) => ({ isDeafened: !state.isDeafened })),
toggleCamera: () => set((state) => ({ isCameraOn: !state.isCameraOn })),
@@ -191,6 +208,8 @@ export const useVoiceStore = create<VoiceState>()(
isLiveKitConnected: false,
inputVolume: 100,
outputVolume: 100,
inputDeviceId: 'default',
outputDeviceId: 'default',
focusedParticipantId: null,
participantVolumes: new Map(),
incomingCall: null,
@@ -210,6 +229,8 @@ export const useVoiceStore = create<VoiceState>()(
isDeafened: state.isDeafened,
inputVolume: state.inputVolume,
outputVolume: state.outputVolume,
inputDeviceId: state.inputDeviceId,
outputDeviceId: state.outputDeviceId,
videoQuality: state.videoQuality,
noiseSuppression: state.noiseSuppression,
}),