devsyncwrld/backspace
1
0
Fork 0
Code Issues Pull Requests Actions 26 Packages Projects Releases Wiki Activity

fix: client-side speaking detection replacing LiveKit's conservative server VAD

Browse Source 
LiveKit's server-side VAD has a high, non-configurable threshold that misses
conversational speech. Replace it with a SpeakingDetector singleton that uses
Web Audio AnalyserNodes to read actual RMS levels per participant (50ms poll,
0.008 threshold, 250ms hysteresis hold).
This commit is contained in:
Jannis Braun
2026-02-23 02:53:40 +01:00
parent 9091f08ced
commit 82eee04946
2 changed files with 267 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/web/src/hooks/useLiveKit.ts
+14 -46
View File
@@ -17,19 +17,12 @@ import {
import { api } from '../api/client';
import { useVoiceStore } from '../stores/voiceStore';
import { AudioManager } from '../audio/AudioManager';
import { SpeakingDetector } from '../audio/SpeakingDetector';
/**
* OPENCORD NATIVE OVERDRIVE PIPELINE v32
* OPENCORD NATIVE OVERDRIVE PIPELINE v33
*/
function setsEqual(a: Set<string>, b: Set<string>): boolean {
if (a.size !== b.size) return false;
for (const v of a) {
if (!b.has(v)) return false;
}
return true;
}
const QUALITY_MAP: Record<string, VideoPreset> = {
'1080p60': new VideoPreset(1920, 1080, 12_000_000, 60),
'1080p': new VideoPreset(1920, 1080, 8_000_000, 30),
@@ -233,6 +226,7 @@ export function useLiveKit() {
processParticipant(r.localParticipant, true);
r.remoteParticipants.forEach((p) => processParticipant(p, false));
useVoiceStore.getState().setParticipants(allParticipants);
SpeakingDetector.getInstance().syncTracks(allParticipants);
}, []);
const handleDataReceived = useCallback((payload: Uint8Array, participant?: RemoteParticipant) => {
@@ -331,6 +325,7 @@ export function useLiveKit() {
await AudioManager.getInstance().resumeContext();
// 1. Reset state immediately to reflect "Loading/Switching" in UI
SpeakingDetector.getInstance().clear();
setRoom(null);
useVoiceStore.getState().setParticipants([]);
useVoiceStore.getState().setSpeakingParticipants(new Set());
@@ -409,12 +404,6 @@ export function useLiveKit() {
});
newRoom.on(RoomEvent.TrackMuted, guardedUpdate);
newRoom.on(RoomEvent.TrackUnmuted, guardedUpdate);
newRoom.on(RoomEvent.ActiveSpeakersChanged, (speakers: Participant[]) => {
if (roomRef.current !== newRoom) return;
useVoiceStore.getState().setSpeakingParticipants(
new Set(speakers.map(s => s.identity))
);
});
newRoom.on(RoomEvent.ParticipantMetadataChanged, guardedUpdate);
newRoom.on(RoomEvent.TrackPublished, (publication: RemoteTrackPublication, participant: RemoteParticipant) => {
if (
@@ -459,6 +448,7 @@ export function useLiveKit() {
});
newRoom.on(RoomEvent.Disconnected, () => {
if (roomRef.current !== newRoom) return;
SpeakingDetector.getInstance().clear();
setConnectionState(ConnectionState.Disconnected);
setConnectedChannelId(null);
roomRef.current = null; _activeRoom = null; setIsConnected(false); setRoom(null);
@@ -510,6 +500,7 @@ export function useLiveKit() {
await AudioManager.getInstance().resumeContext();
// 1. Reset state immediately
SpeakingDetector.getInstance().clear();
setRoom(null);
useVoiceStore.getState().setParticipants([]);
useVoiceStore.getState().setSpeakingParticipants(new Set());
@@ -572,12 +563,6 @@ export function useLiveKit() {
});
newRoom.on(RoomEvent.TrackMuted, guardedUpdate);
newRoom.on(RoomEvent.TrackUnmuted, guardedUpdate);
newRoom.on(RoomEvent.ActiveSpeakersChanged, (speakers: Participant[]) => {
if (roomRef.current !== newRoom) return;
useVoiceStore.getState().setSpeakingParticipants(
new Set(speakers.map(s => s.identity))
);
});
newRoom.on(RoomEvent.TrackPublished, (publication: RemoteTrackPublication, participant: RemoteParticipant) => {
if (
publication.source === Track.Source.ScreenShare ||
@@ -654,14 +639,15 @@ export function useLiveKit() {
const disconnect = useCallback(async () => {
_connectGeneration++;
SpeakingDetector.getInstance().clear();
connectedChannelRef.current = null;
setConnectedChannelId(null);
if (roomRef.current) {
await roomRef.current.disconnect();
roomRef.current = null;
_activeRoom = null;
setRoom(null);
setIsConnected(false);
if (roomRef.current) {
await roomRef.current.disconnect();
roomRef.current = null;
_activeRoom = null;
setRoom(null);
setIsConnected(false);
setIsConnecting(false);
setConnectionState(ConnectionState.Disconnected);
useVoiceStore.getState().setParticipants([]);
@@ -775,27 +761,9 @@ export function useLiveKit() {
}, [room]);
useEffect(() => {
return () => { _connectGeneration++; if (roomRef.current) { roomRef.current.disconnect(); roomRef.current = null; _activeRoom = null; } };
return () => { _connectGeneration++; SpeakingDetector.getInstance().clear(); if (roomRef.current) { roomRef.current.disconnect(); roomRef.current = null; _activeRoom = null; } };
}, []);
// Speaking poll safety net: catches missed ActiveSpeakersChanged events
useEffect(() => {
if (!isConnected) return;
const interval = setInterval(() => {
const r = roomRef.current;
if (!r) return;
const speakingIds = new Set<string>();
if (r.localParticipant.isSpeaking) speakingIds.add(r.localParticipant.identity);
r.remoteParticipants.forEach((p) => {
if (p.isSpeaking) speakingIds.add(p.identity);
});
const current = useVoiceStore.getState().speakingParticipantIds;
if (!setsEqual(current, speakingIds)) {
useVoiceStore.getState().setSpeakingParticipants(speakingIds);
}
}, 200);
return () => clearInterval(interval);
}, [isConnected]);
return { room, isConnected, isConnecting, connectionState, connectedChannelId, connectionError, connect, connectDm, disconnect, toggleMic, toggleCamera, toggleScreenShare };
}