devsyncwrld/backspace
1
0
Fork 0
Code Issues Pull Requests Actions 23 Packages Projects Releases Wiki Activity

feat: RNNoise WASM AudioWorklet for ML-based noise suppression (Phase 3)

Browse Source 
Inject @sapphi-red/web-noise-suppressor into AudioManager's input pipeline
as a toggleable AudioWorkletNode. The worklet is loaded lazily on first
enable, then kept alive — toggling bypasses by rewiring the graph without
destroying the WASM instance. Browser NS is forced off when RNNoise is
active to avoid double-processing. InputGain forced to stereo up-mix to
prevent mono-left-only output from the worklet. Also wires Phase 2 output
device routing (setSinkId) into AppLayout/ChannelSidebar.
This commit is contained in:
Jannis Braun
2026-02-23 00:28:29 +01:00
parent dbebd38576
commit ccf2001047
15 changed files with 345 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/web/package.json
+1
View File
@@ -11,6 +11,7 @@
"dependencies": {
"@livekit/components-react": "^2.7.4",
"@opencord/shared": "workspace:*",
"@sapphi-red/web-noise-suppressor": "^0.3.5",
"livekit-client": "^2.9.0",
"react": "^18.3.1",
"react-dom": "^18.3.1",
packages/web/src/audio/AudioManager.js
+109 -11
View File
@@ -1,3 +1,8 @@
import { RnnoiseWorkletNode, loadRnnoise } from '@sapphi-red/web-noise-suppressor';
import rnnoiseWorkletPath from '@sapphi-red/web-noise-suppressor/rnnoiseWorklet.js?url';
import rnnoiseWasmPath from '@sapphi-red/web-noise-suppressor/rnnoise.wasm?url';
import rnnoiseWasmSimdPath from '@sapphi-red/web-noise-suppressor/rnnoise_simd.wasm?url';
export class AudioManager {
static instance = null;
ctx = null;
@@ -8,6 +13,7 @@ export class AudioManager {
analyser = null;
masterCompressor = null;
currentInputDeviceId = 'default';
desiredOutputDeviceId = 'default';
currentStream = null;
isInitialized = false;
listeners = new Set();
@@ -15,7 +21,12 @@ export class AudioManager {
voiceEchoCancellation = true;
voiceNoiseSuppression = true;
voiceAutoGainControl = false;
screenShareActive = false;
streamGeneration = 0;
inputSwitchChain = Promise.resolve(null);
rnnoiseNode = null;
rnnoiseEnabled = false;
rnnoiseReady = false;
constructor() { }
static getInstance() {
if (!AudioManager.instance) {
@@ -27,8 +38,13 @@ export class AudioManager {
if (this.ctx)
return;
const AudioContextClass = window.AudioContext || window.webkitAudioContext;
this.ctx = new AudioContextClass();
this.ctx = new AudioContextClass({ sampleRate: 48000 });
this.inputGain = this.ctx.createGain();
// Force stereo up-mix so mono sources (e.g. RNNoise worklet output)
// are duplicated to both L+R channels instead of left-only.
this.inputGain.channelCount = 2;
this.inputGain.channelCountMode = 'explicit';
this.inputGain.channelInterpretation = 'speakers';
this.inputDestination = this.ctx.createMediaStreamDestination();
this.analyser = this.ctx.createAnalyser();
this.analyser.fftSize = 256;
@@ -55,6 +71,22 @@ export class AudioManager {
}
};
this.isInitialized = true;
this.applyOutputDevice();
}
async setOutputDevice(deviceId) {
this.desiredOutputDeviceId = deviceId;
await this.applyOutputDevice();
}
async applyOutputDevice() {
if (!this.ctx || !('setSinkId' in this.ctx)) return;
try {
const sinkId = this.desiredOutputDeviceId === 'default' ? '' : this.desiredOutputDeviceId;
await this.ctx.setSinkId(sinkId);
console.log(`[AudioManager] Output device set to: ${this.desiredOutputDeviceId}`);
}
catch (err) {
console.error('[AudioManager] Failed to set output device:', err);
}
}
onResumed(cb) {
this.listeners.add(cb);
@@ -112,6 +144,11 @@ export class AudioManager {
return source;
}
async setInputDevice(deviceId) {
const job = this.inputSwitchChain.then(() => this._setInputDeviceImpl(deviceId));
this.inputSwitchChain = job.catch(() => null);
return job;
}
async _setInputDeviceImpl(deviceId) {
if (!this.isInitialized)
this.initContext();
// Skip if already set and stream is active
@@ -122,12 +159,20 @@ export class AudioManager {
if (this.currentStream) {
this.currentStream.getTracks().forEach(t => t.stop());
}
const effectiveEchoCancellation = this.screenShareActive ? false : this.voiceEchoCancellation;
// When RNNoise is active, force browser NS off — running both degrades quality.
const effectiveNoiseSuppression = this.rnnoiseEnabled ? false : this.voiceNoiseSuppression;
const constraints = {
audio: {
deviceId: deviceId === 'default' ? undefined : { exact: deviceId },
echoCancellation: this.voiceEchoCancellation,
noiseSuppression: this.voiceNoiseSuppression,
echoCancellation: effectiveEchoCancellation,
noiseSuppression: effectiveNoiseSuppression,
autoGainControl: this.voiceAutoGainControl,
googEchoCancellation: effectiveEchoCancellation,
googAutoGainControl: this.voiceAutoGainControl,
googNoiseSuppression: effectiveNoiseSuppression,
googHighpassFilter: false,
googTypingNoiseDetection: false,
}
};
this.currentStream = await navigator.mediaDevices.getUserMedia(constraints);
@@ -138,7 +183,7 @@ export class AudioManager {
this.inputSource.disconnect();
}
this.inputSource = this.ctx.createMediaStreamSource(this.currentStream);
this.inputSource.connect(this.inputGain);
this.inputSource.connect(this.getInputTarget());
}
return this.currentStream;
}
@@ -147,6 +192,56 @@ export class AudioManager {
throw err;
}
}
getInputTarget() {
return (this.rnnoiseEnabled && this.rnnoiseNode) ? this.rnnoiseNode : this.inputGain;
}
async setRnnoiseEnabled(enabled) {
if (enabled === this.rnnoiseEnabled && this.rnnoiseReady)
return;
if (!this.isInitialized)
this.initContext();
if (enabled && !this.rnnoiseReady) {
try {
console.log('[AudioManager] Loading RNNoise worklet...');
await this.ctx.audioWorklet.addModule(rnnoiseWorkletPath);
// loadRnnoise handles SIMD feature detection and returns the right binary
const wasmBinary = await loadRnnoise({ url: rnnoiseWasmPath, simdUrl: rnnoiseWasmSimdPath });
this.rnnoiseNode = new RnnoiseWorkletNode(this.ctx, {
wasmBinary,
maxChannels: 1,
});
this.rnnoiseNode.connect(this.inputGain);
this.rnnoiseReady = true;
console.log('[AudioManager] RNNoise worklet loaded and connected');
}
catch (err) {
console.error('[AudioManager] Failed to load RNNoise worklet:', err);
this.rnnoiseReady = false;
this.rnnoiseEnabled = false;
if (this.inputSource && this.inputGain) {
this.inputSource.disconnect();
this.inputSource.connect(this.inputGain);
}
return;
}
}
this.rnnoiseEnabled = enabled;
// Rewire the graph
if (this.inputSource) {
this.inputSource.disconnect();
this.inputSource.connect(this.getInputTarget());
console.log(`[AudioManager] RNNoise ${enabled ? 'enabled' : 'bypassed'} — inputSource → ${enabled ? 'rnnoiseNode' : 'inputGain'}`);
}
// Force track re-publish so LiveKit picks up the new pipeline
this.streamGeneration++;
if (this.currentStream) {
this.currentStream.getTracks().forEach(t => t.stop());
this.currentStream = null;
}
}
isRnnoiseEnabled() {
return this.rnnoiseEnabled;
}
setInputVolume(volume) {
if (!this.isInitialized)
this.initContext();
@@ -174,6 +269,15 @@ export class AudioManager {
this.currentStream = null;
}
}
setScreenShareActive(active) {
if (this.screenShareActive === active) return;
this.screenShareActive = active;
console.log(`[AudioManager] Screen share active: ${active} — ${active ? 'forcing AEC off' : 'restoring user AEC preference'}`);
if (this.currentStream) {
this.currentStream.getTracks().forEach(t => t.stop());
this.currentStream = null;
}
}
getStreamGeneration() {
return this.streamGeneration;
}
@@ -208,16 +312,10 @@ export class AudioManager {
this.initContext();
return this.ctx;
}
/**
* Returns the master output bus (DynamicsCompressorNode).
* All remote audio (voice, stream) should connect their GainNodes
* to this node instead of directly to ctx.destination. The compressor
* prevents clipping when multiple sources sum together.
*/
getMasterOutput() {
if (!this.ctx)
this.initContext();
return this.ctx.destination;
return this.masterCompressor;
}
getContext() {
return this.ctx;
packages/web/src/audio/AudioManager.ts
+125 -12
View File
@@ -1,3 +1,8 @@
import { RnnoiseWorkletNode, loadRnnoise } from '@sapphi-red/web-noise-suppressor';
import rnnoiseWorkletPath from '@sapphi-red/web-noise-suppressor/rnnoiseWorklet.js?url';
import rnnoiseWasmPath from '@sapphi-red/web-noise-suppressor/rnnoise.wasm?url';
import rnnoiseWasmSimdPath from '@sapphi-red/web-noise-suppressor/rnnoise_simd.wasm?url';
export class AudioManager {
private static instance: AudioManager | null = null;
private ctx: AudioContext | null = null;
@@ -9,6 +14,7 @@ export class AudioManager {
private masterCompressor: DynamicsCompressorNode | null = null;
private currentInputDeviceId: string = 'default';
private desiredOutputDeviceId: string = 'default';
private currentStream: MediaStream | null = null;
private isInitialized = false;
@@ -19,6 +25,10 @@ export class AudioManager {
private voiceAutoGainControl = false;
private screenShareActive = false;
private streamGeneration = 0;
private inputSwitchChain: Promise<MediaStream | null> = Promise.resolve(null);
private rnnoiseNode: AudioWorkletNode | null = null;
private rnnoiseEnabled = false;
private rnnoiseReady = false;
private constructor() {}
@@ -32,9 +42,14 @@ export class AudioManager {
private initContext() {
if (this.ctx) return;
const AudioContextClass = window.AudioContext || (window as any).webkitAudioContext;
this.ctx = new AudioContextClass();
this.ctx = new AudioContextClass({ sampleRate: 48000 });
this.inputGain = this.ctx.createGain();
// Force stereo up-mix so mono sources (e.g. RNNoise worklet output)
// are duplicated to both L+R channels instead of left-only.
this.inputGain.channelCount = 2;
this.inputGain.channelCountMode = 'explicit';
this.inputGain.channelInterpretation = 'speakers';
this.inputDestination = this.ctx.createMediaStreamDestination();
this.analyser = this.ctx.createAnalyser();
this.analyser.fftSize = 256;
@@ -67,6 +82,31 @@ export class AudioManager {
};
this.isInitialized = true;
// Apply pending output device selection (user preference loaded before context creation)
this.applyOutputDevice();
}
/**
* Routes all Web Audio output to the specified device via AudioContext.setSinkId().
* This is the ONLY correct way to switch output devices when using a custom Web Audio
* pipeline — LiveKit's switchActiveDevice('audiooutput') targets <audio> elements
* which we deliberately kill via MutationObserver.
*/
async setOutputDevice(deviceId: string): Promise<void> {
this.desiredOutputDeviceId = deviceId;
await this.applyOutputDevice();
}
private async applyOutputDevice(): Promise<void> {
if (!this.ctx || !('setSinkId' in this.ctx)) return;
try {
const sinkId = this.desiredOutputDeviceId === 'default' ? '' : this.desiredOutputDeviceId;
await (this.ctx as any).setSinkId(sinkId);
console.log(`[AudioManager] Output device set to: ${this.desiredOutputDeviceId}`);
} catch (err) {
console.error('[AudioManager] Failed to set output device:', err);
}
}
onResumed(cb: () => void) {
@@ -129,9 +169,21 @@ export class AudioManager {
return source;
}
async setInputDevice(deviceId: string) {
/**
* Serialized input device switch.
* Multiple callers (store, syncMic, UI) may trigger this concurrently.
* Chaining ensures only one getUserMedia runs at a time, and the second
* call short-circuits if the first already set the same device.
*/
async setInputDevice(deviceId: string): Promise<MediaStream | null> {
const job = this.inputSwitchChain.then(() => this._setInputDeviceImpl(deviceId));
this.inputSwitchChain = job.catch(() => null);
return job;
}
private async _setInputDeviceImpl(deviceId: string): Promise<MediaStream | null> {
if (!this.isInitialized) this.initContext();
// Skip if already set and stream is active
if (this.currentInputDeviceId === deviceId && this.currentStream?.active) {
return this.currentStream;
@@ -147,17 +199,21 @@ export class AudioManager {
// Force AEC off during screen share to prevent this.
const effectiveEchoCancellation = this.screenShareActive ? false : this.voiceEchoCancellation;
// When RNNoise is active, force browser NS off — running both degrades quality.
// The user's noiseSuppression preference is preserved in the store for when RNNoise is disabled.
const effectiveNoiseSuppression = this.rnnoiseEnabled ? false : this.voiceNoiseSuppression;
const constraints = {
audio: {
deviceId: deviceId === 'default' ? undefined : { exact: deviceId },
echoCancellation: effectiveEchoCancellation,
noiseSuppression: this.voiceNoiseSuppression,
noiseSuppression: effectiveNoiseSuppression,
autoGainControl: this.voiceAutoGainControl,
// Chromium-specific constraints — belt-and-suspenders to ensure
// Chrome's internal audio engine respects the standard constraints.
googEchoCancellation: effectiveEchoCancellation,
googAutoGainControl: this.voiceAutoGainControl,
googNoiseSuppression: this.voiceNoiseSuppression,
googNoiseSuppression: effectiveNoiseSuppression,
googHighpassFilter: false,
googTypingNoiseDetection: false,
} as any
@@ -172,9 +228,9 @@ export class AudioManager {
this.inputSource.disconnect();
}
this.inputSource = this.ctx.createMediaStreamSource(this.currentStream);
this.inputSource.connect(this.inputGain);
this.inputSource.connect(this.getInputTarget());
}
return this.currentStream;
} catch (err) {
console.error('[AudioManager] Failed to set input device:', err);
@@ -182,6 +238,63 @@ export class AudioManager {
}
}
private getInputTarget(): AudioNode {
return (this.rnnoiseEnabled && this.rnnoiseNode) ? this.rnnoiseNode : this.inputGain!;
}
async setRnnoiseEnabled(enabled: boolean): Promise<void> {
if (enabled === this.rnnoiseEnabled && this.rnnoiseReady) return;
if (!this.isInitialized) this.initContext();
if (enabled && !this.rnnoiseReady) {
try {
console.log('[AudioManager] Loading RNNoise worklet...');
await this.ctx!.audioWorklet.addModule(rnnoiseWorkletPath);
// loadRnnoise handles SIMD feature detection and returns the right binary
const wasmBinary = await loadRnnoise({ url: rnnoiseWasmPath, simdUrl: rnnoiseWasmSimdPath });
this.rnnoiseNode = new RnnoiseWorkletNode(this.ctx!, {
wasmBinary,
maxChannels: 1,
});
this.rnnoiseNode.connect(this.inputGain!);
this.rnnoiseReady = true;
console.log('[AudioManager] RNNoise worklet loaded and connected');
} catch (err) {
console.error('[AudioManager] Failed to load RNNoise worklet:', err);
this.rnnoiseReady = false;
this.rnnoiseEnabled = false;
// Fall back to direct wiring
if (this.inputSource && this.inputGain) {
this.inputSource.disconnect();
this.inputSource.connect(this.inputGain);
}
return;
}
}
this.rnnoiseEnabled = enabled;
// Rewire the graph
if (this.inputSource) {
this.inputSource.disconnect();
this.inputSource.connect(this.getInputTarget());
console.log(`[AudioManager] RNNoise ${enabled ? 'enabled' : 'bypassed'} — inputSource → ${enabled ? 'rnnoiseNode' : 'inputGain'}`);
}
// Force track re-publish so LiveKit picks up the new pipeline
this.streamGeneration++;
if (this.currentStream) {
this.currentStream.getTracks().forEach(t => t.stop());
this.currentStream = null;
}
}
isRnnoiseEnabled(): boolean {
return this.rnnoiseEnabled;
}
setInputVolume(volume: number) {
if (!this.isInitialized) this.initContext();
if (this.inputGain && this.ctx) {
@@ -261,14 +374,14 @@ export class AudioManager {
}
/**
* Returns the master output bus (DynamicsCompressorNode).
* All remote audio (voice, stream) should connect their GainNodes
* to this node instead of directly to ctx.destination. The compressor
* prevents clipping when multiple sources sum together.
* Returns the master output bus (DynamicsCompressorNode → ctx.destination).
* All audio (remote voice, streams, effects) routes through this node.
* The compressor prevents clipping when multiple sources sum together.
* Output device is controlled via setSinkId on the underlying AudioContext.
*/
getMasterOutput(): AudioNode {
if (!this.ctx) this.initContext();
return this.ctx!.destination;
return this.masterCompressor!;
}
getContext(): AudioContext | null {
packages/web/src/components/layout/AppLayout.js
+4
View File
@@ -65,6 +65,10 @@ export function AppLayout() {
observer.observe(document.body, { childList: true, subtree: true });
return () => observer.disconnect();
}, []);
const outputDeviceId = useVoiceStore((s) => s.outputDeviceId);
useEffect(() => {
AudioManager.getInstance().setOutputDevice(outputDeviceId);
}, [outputDeviceId]);
const { user, isLoading } = useAuth();
const setCurrentServer = useServerStore((s) => s.setCurrentServer);
const loadServerDetail = useServerStore((s) => s.loadServerDetail);
packages/web/src/components/layout/AppLayout.tsx
+8
View File
@@ -69,6 +69,14 @@ export function AppLayout() {
return () => observer.disconnect();
}, []);
// Sync persisted output device preference to AudioManager.
// AudioManager defers setSinkId until the AudioContext is actually created,
// so this is safe to call before any user interaction.
const outputDeviceId = useVoiceStore((s) => s.outputDeviceId);
useEffect(() => {
AudioManager.getInstance().setOutputDevice(outputDeviceId);
}, [outputDeviceId]);
const { user, isLoading } = useAuth();
const setCurrentServer = useServerStore((s) => s.setCurrentServer);
const loadServerDetail = useServerStore((s) => s.loadServerDetail);
packages/web/src/components/layout/ChannelSidebar.js
+12 -5
View File
@@ -144,8 +144,16 @@ function UserAreaPanel({ user, isMuted, isDeafened, onMicToggle, onDeafenToggle,
await navigator.mediaDevices.getUserMedia({ audio: true }).then(s => s.getTracks().forEach(t => t.stop()));
}
const devices = await navigator.mediaDevices.enumerateDevices();
const inputs = devices.filter(d => d.kind === 'audioinput');
const outputs = devices.filter(d => d.kind === 'audiooutput');
const dedup = (list) => {
const seen = new Set();
return list.filter(d => {
if (seen.has(d.deviceId)) return false;
seen.add(d.deviceId);
return true;
});
};
const inputs = dedup(devices.filter(d => d.kind === 'audioinput'));
const outputs = dedup(devices.filter(d => d.kind === 'audiooutput'));
setInputDevices(inputs);
setOutputDevices(outputs);
const currentInput = inputs.find(d => d.deviceId === inputDeviceId);
@@ -220,6 +228,7 @@ function UserAreaPanel({ user, isMuted, isDeafened, onMicToggle, onDeafenToggle,
};
const selectInput = (device) => {
setInputDevice(device.deviceId);
AudioManager.getInstance().setInputDevice(device.deviceId);
setSelectedInputLabel(device.label || 'Default');
setShowInputDeviceList(false);
};
@@ -227,9 +236,7 @@ function UserAreaPanel({ user, isMuted, isDeafened, onMicToggle, onDeafenToggle,
setOutputDevice(device.deviceId);
setSelectedOutputLabel(device.label || 'Default');
setShowOutputDeviceList(false);
const room = getActiveRoom();
if (room)
room.switchActiveDevice('audiooutput', device.deviceId).catch(() => { });
AudioManager.getInstance().setOutputDevice(device.deviceId);
};
// Generate mic level bars (20 bars like Discord)
const micBars = 20;
packages/web/src/components/layout/ChannelSidebar.tsx
+15 -5
View File
@@ -408,8 +408,18 @@ function UserAreaPanel({
await navigator.mediaDevices.getUserMedia({ audio: true }).then(s => s.getTracks().forEach(t => t.stop()));
}
const devices = await navigator.mediaDevices.enumerateDevices();
const inputs = devices.filter(d => d.kind === 'audioinput');
const outputs = devices.filter(d => d.kind === 'audiooutput');
// Deduplicate by deviceId — USB devices sharing the same audio chipset
// (e.g. C-Media 0d8c:0134) appear as multiple entries with identical IDs.
const dedup = (list: MediaDeviceInfo[]): MediaDeviceInfo[] => {
const seen = new Set<string>();
return list.filter(d => {
if (seen.has(d.deviceId)) return false;
seen.add(d.deviceId);
return true;
});
};
const inputs = dedup(devices.filter(d => d.kind === 'audioinput'));
const outputs = dedup(devices.filter(d => d.kind === 'audiooutput'));
setInputDevices(inputs);
setOutputDevices(outputs);
@@ -483,7 +493,8 @@ function UserAreaPanel({
};
const selectInput = (device: MediaDeviceInfo) => {
setInputDevice(device.deviceId);
setInputDevice(device.deviceId); // Pure state update → triggers syncMic if in voice call
AudioManager.getInstance().setInputDevice(device.deviceId); // Immediate preview for mic level meter
setSelectedInputLabel(device.label || 'Default');
setShowInputDeviceList(false);
};
@@ -492,8 +503,7 @@ function UserAreaPanel({
setOutputDevice(device.deviceId);
setSelectedOutputLabel(device.label || 'Default');
setShowOutputDeviceList(false);
const room = getActiveRoom();
if (room) room.switchActiveDevice('audiooutput', device.deviceId).catch(() => {});
AudioManager.getInstance().setOutputDevice(device.deviceId);
};
// Generate mic level bars (20 bars like Discord)
packages/web/src/components/voice/VoiceControls.js
+5 -1
View File
@@ -17,6 +17,8 @@ export function VoiceControls() {
const toggleScreenShare = useVoiceStore((s) => s.toggleScreenShare);
const noiseSuppression = useVoiceStore((s) => s.noiseSuppression);
const toggleNoiseSuppression = useVoiceStore((s) => s.toggleNoiseSuppression);
const rnnoiseEnabled = useVoiceStore((s) => s.rnnoiseEnabled);
const toggleRnnoise = useVoiceStore((s) => s.toggleRnnoise);
const connectionError = useVoiceStore((s) => s.connectionError);
const isLiveKitConnected = useVoiceStore((s) => s.isLiveKitConnected);
const channels = useServerStore((s) => s.channels);
@@ -81,5 +83,7 @@ export function VoiceControls() {
? 'bg-[#111214] text-discord-blurple hover:bg-[#1a1b1e]'
: btnDefaultStyle}`, title: "Video Quality", children: _jsxs("svg", { width: "20", height: "20", viewBox: "0 0 24 24", fill: "currentColor", children: [_jsx("path", { d: "M3 5v14h18V5H3zm16 12H5V7h14v10z" }), _jsx("path", { d: "M8 15l2.5-3.21L13 15l2-2.5L18 17H6z" })] }) }), _jsx("button", { onClick: handleNoiseSuppression, className: `${btnBase} ${noiseSuppression
? 'bg-[#111214] text-discord-green hover:bg-[#1a1b1e]'
: btnDefaultStyle}`, title: noiseSuppression ? 'Disable Noise Suppression' : 'Enable Noise Suppression', children: _jsxs("svg", { width: "20", height: "20", viewBox: "0 0 24 24", fill: "currentColor", children: [_jsx("path", { d: "M7 9v6h4l5 5V4l-5 5H7z" }), noiseSuppression ? (_jsxs(_Fragment, { children: [_jsx("path", { d: "M19 12c0-1.66-.68-3.16-1.76-4.24l-1.42 1.42C16.55 9.9 17 10.9 17 12c0 1.1-.45 2.1-1.18 2.82l1.42 1.42C18.32 15.16 19 13.66 19 12z" }), _jsx("path", { d: "M21 12c0-2.76-1.12-5.26-2.93-7.07l-1.42 1.42C18.2 7.9 19 9.85 19 12c0 2.15-.8 4.1-2.35 5.65l1.42 1.42C19.88 17.26 21 14.76 21 12z", opacity: "0.6" })] })) : (_jsx("line", { x1: "19", y1: "5", x2: "19", y2: "19", stroke: "currentColor", strokeWidth: "2", strokeLinecap: "round", opacity: "0.4" }))] }) }), _jsx(VideoQualityPopover, { open: showVideoQuality, onClose: () => setShowVideoQuality(false) })] })] }));
: btnDefaultStyle}`, title: noiseSuppression ? 'Disable Browser Noise Suppression' : 'Enable Browser Noise Suppression', children: _jsxs("svg", { width: "20", height: "20", viewBox: "0 0 24 24", fill: "currentColor", children: [_jsx("path", { d: "M7 9v6h4l5 5V4l-5 5H7z" }), noiseSuppression ? (_jsxs(_Fragment, { children: [_jsx("path", { d: "M19 12c0-1.66-.68-3.16-1.76-4.24l-1.42 1.42C16.55 9.9 17 10.9 17 12c0 1.1-.45 2.1-1.18 2.82l1.42 1.42C18.32 15.16 19 13.66 19 12z" }), _jsx("path", { d: "M21 12c0-2.76-1.12-5.26-2.93-7.07l-1.42 1.42C18.2 7.9 19 9.85 19 12c0 2.15-.8 4.1-2.35 5.65l1.42 1.42C19.88 17.26 21 14.76 21 12z", opacity: "0.6" })] })) : (_jsx("line", { x1: "19", y1: "5", x2: "19", y2: "19", stroke: "currentColor", strokeWidth: "2", strokeLinecap: "round", opacity: "0.4" }))] }) }), _jsx("button", { onClick: toggleRnnoise, className: `${btnBase} ${rnnoiseEnabled
? 'bg-[#111214] text-discord-green hover:bg-[#1a1b1e]'
: btnDefaultStyle}`, title: rnnoiseEnabled ? 'Disable AI Noise Suppression' : 'Enable AI Noise Suppression', children: _jsxs("svg", { width: "20", height: "20", viewBox: "0 0 24 24", fill: "currentColor", children: [_jsx("path", { d: "M12 2C6.48 2 2 6.48 2 12s4.48 10 10 10 10-4.48 10-10S17.52 2 12 2z", opacity: rnnoiseEnabled ? 0.15 : 0.08 }), _jsx("path", { d: "M12 1a2 2 0 012 2v1a2 2 0 01-4 0V3a2 2 0 012-2z" }), _jsx("path", { d: "M12 7c-1.66 0-3 1.34-3 3v2c0 1.66 1.34 3 3 3s3-1.34 3-3v-2c0-1.66-1.34-3-3-3z" }), _jsx("path", { d: "M17 11v1c0 2.76-2.24 5-5 5s-5-2.24-5-5v-1H5v1c0 3.53 2.61 6.43 6 6.92V21h2v-2.08c3.39-.49 6-3.39 6-6.92v-1h-2z" }), rnnoiseEnabled ? (_jsxs(_Fragment, { children: [_jsx("circle", { cx: "18", cy: "5", r: "1.2", fill: "currentColor" }), _jsx("circle", { cx: "20", cy: "8", r: "0.9", fill: "currentColor", opacity: "0.7" }), _jsx("circle", { cx: "6", cy: "5", r: "1.2", fill: "currentColor" }), _jsx("circle", { cx: "4", cy: "8", r: "0.9", fill: "currentColor", opacity: "0.7" })] })) : (_jsx("line", { x1: "4", y1: "4", x2: "20", y2: "20", stroke: "currentColor", strokeWidth: "2", strokeLinecap: "round", opacity: "0.4" }))] }) }), _jsx(VideoQualityPopover, { open: showVideoQuality, onClose: () => setShowVideoQuality(false) })] })] }));
}
packages/web/src/components/voice/VoiceControls.tsx
+32 -2
View File
@@ -18,6 +18,8 @@ export function VoiceControls() {
const toggleScreenShare = useVoiceStore((s) => s.toggleScreenShare);
const noiseSuppression = useVoiceStore((s) => s.noiseSuppression);
const toggleNoiseSuppression = useVoiceStore((s) => s.toggleNoiseSuppression);
const rnnoiseEnabled = useVoiceStore((s) => s.rnnoiseEnabled);
const toggleRnnoise = useVoiceStore((s) => s.toggleRnnoise);
const connectionError = useVoiceStore((s) => s.connectionError);
const isLiveKitConnected = useVoiceStore((s) => s.isLiveKitConnected);
const channels = useServerStore((s) => s.channels);
@@ -172,7 +174,7 @@ export function VoiceControls() {
</svg>
</button>
{/* Noise Suppression */}
{/* Browser Noise Suppression */}
<button
onClick={handleNoiseSuppression}
className={`${btnBase} ${
@@ -180,7 +182,7 @@ export function VoiceControls() {
? 'bg-[#111214] text-discord-green hover:bg-[#1a1b1e]'
: btnDefaultStyle
}`}
title={noiseSuppression ? 'Disable Noise Suppression' : 'Enable Noise Suppression'}
title={noiseSuppression ? 'Disable Browser Noise Suppression' : 'Enable Browser Noise Suppression'}
>
<svg width="20" height="20" viewBox="0 0 24 24" fill="currentColor">
<path d="M7 9v6h4l5 5V4l-5 5H7z" />
@@ -195,6 +197,34 @@ export function VoiceControls() {
</svg>
</button>
{/* AI Noise Suppression (RNNoise) */}
<button
onClick={toggleRnnoise}
className={`${btnBase} ${
rnnoiseEnabled
? 'bg-[#111214] text-discord-green hover:bg-[#1a1b1e]'
: btnDefaultStyle
}`}
title={rnnoiseEnabled ? 'Disable AI Noise Suppression' : 'Enable AI Noise Suppression'}
>
<svg width="20" height="20" viewBox="0 0 24 24" fill="currentColor">
<path d="M12 2C6.48 2 2 6.48 2 12s4.48 10 10 10 10-4.48 10-10S17.52 2 12 2z" opacity={rnnoiseEnabled ? 0.15 : 0.08} />
<path d="M12 1a2 2 0 012 2v1a2 2 0 01-4 0V3a2 2 0 012-2z" />
<path d="M12 7c-1.66 0-3 1.34-3 3v2c0 1.66 1.34 3 3 3s3-1.34 3-3v-2c0-1.66-1.34-3-3-3z" />
<path d="M17 11v1c0 2.76-2.24 5-5 5s-5-2.24-5-5v-1H5v1c0 3.53 2.61 6.43 6 6.92V21h2v-2.08c3.39-.49 6-3.39 6-6.92v-1h-2z" />
{rnnoiseEnabled ? (
<>
<circle cx="18" cy="5" r="1.2" fill="currentColor" />
<circle cx="20" cy="8" r="0.9" fill="currentColor" opacity="0.7" />
<circle cx="6" cy="5" r="1.2" fill="currentColor" />
<circle cx="4" cy="8" r="0.9" fill="currentColor" opacity="0.7" />
</>
) : (
<line x1="4" y1="4" x2="20" y2="20" stroke="currentColor" strokeWidth="2" strokeLinecap="round" opacity="0.4" />
)}
</svg>
</button>
{/* Video Quality Popover */}
<VideoQualityPopover
open={showVideoQuality}
packages/web/src/hooks/useLiveKit.js
+3 -1
View File
@@ -105,6 +105,7 @@ export function useLiveKit() {
const echoCancellation = useVoiceStore((s) => s.echoCancellation);
const noiseSuppression = useVoiceStore((s) => s.noiseSuppression);
const autoGainControl = useVoiceStore((s) => s.autoGainControl);
const rnnoiseEnabled = useVoiceStore((s) => s.rnnoiseEnabled);
const lastMicGenRef = useRef(0);
const updateParticipants = useCallback(() => {
const r = roomRef.current;
@@ -199,6 +200,7 @@ export function useLiveKit() {
try {
const audioManager = AudioManager.getInstance();
audioManager.setVoiceProcessing({ echoCancellation, noiseSuppression, autoGainControl });
await audioManager.setRnnoiseEnabled(rnnoiseEnabled);
audioManager.setScreenShareActive(isScreenSharing);
const micPub = r.localParticipant.getTrackPublications()
.find(p => p.source === Track.Source.Microphone);
@@ -247,7 +249,7 @@ export function useLiveKit() {
return () => {
unsubscribe();
};
}, [isMuted, isDeafened, inputDeviceId, inputVolume, isConnected, echoCancellation, noiseSuppression, autoGainControl, isScreenSharing]);
}, [isMuted, isDeafened, inputDeviceId, inputVolume, isConnected, echoCancellation, noiseSuppression, autoGainControl, rnnoiseEnabled, isScreenSharing]);
const connect = useCallback(async (channelId) => {
if (connectedChannelRef.current === channelId && roomRef.current?.state === ConnectionState.Connected)
return;
packages/web/src/hooks/useLiveKit.ts
+3 -1
View File
@@ -161,6 +161,7 @@ export function useLiveKit() {
const echoCancellation = useVoiceStore((s) => s.echoCancellation);
const noiseSuppression = useVoiceStore((s) => s.noiseSuppression);
const autoGainControl = useVoiceStore((s) => s.autoGainControl);
const rnnoiseEnabled = useVoiceStore((s) => s.rnnoiseEnabled);
const lastMicGenRef = useRef(0);
@@ -254,6 +255,7 @@ export function useLiveKit() {
// Sync voice processing settings to AudioManager
audioManager.setVoiceProcessing({ echoCancellation, noiseSuppression, autoGainControl });
await audioManager.setRnnoiseEnabled(rnnoiseEnabled);
// Keep screen share state in sync (handles edge cases like remounts)
audioManager.setScreenShareActive(isScreenSharing);
@@ -313,7 +315,7 @@ export function useLiveKit() {
return () => {
unsubscribe();
};
}, [isMuted, isDeafened, inputDeviceId, inputVolume, isConnected, echoCancellation, noiseSuppression, autoGainControl, isScreenSharing]);
}, [isMuted, isDeafened, inputDeviceId, inputVolume, isConnected, echoCancellation, noiseSuppression, autoGainControl, rnnoiseEnabled, isScreenSharing]);
const connect = useCallback(async (channelId: string) => {
if (connectedChannelRef.current === channelId && roomRef.current?.state === ConnectionState.Connected) return;
packages/web/src/stores/voiceStore.js
+8 -5
View File
@@ -119,10 +119,7 @@ export const useVoiceStore = create()(persist((set, get) => ({
AudioManager.getInstance().setInputVolume(volume);
},
setOutputVolume: (volume) => set({ outputVolume: volume }),
setInputDevice: async (deviceId) => {
set({ inputDeviceId: deviceId });
await AudioManager.getInstance().setInputDevice(deviceId);
},
setInputDevice: (deviceId) => set({ inputDeviceId: deviceId }),
setOutputDevice: (deviceId) => set({ outputDeviceId: deviceId }),
toggleMic: () => set((state) => ({ isMuted: !state.isMuted })),
toggleDeafen: () => set((state) => ({ isDeafened: !state.isDeafened })),
@@ -133,9 +130,11 @@ export const useVoiceStore = create()(persist((set, get) => ({
noiseSuppression: true,
echoCancellation: true,
autoGainControl: false,
rnnoiseEnabled: false,
toggleNoiseSuppression: () => set((state) => ({ noiseSuppression: !state.noiseSuppression })),
setEchoCancellation: (enabled) => set({ echoCancellation: enabled }),
setAutoGainControl: (enabled) => set({ autoGainControl: enabled }),
toggleRnnoise: () => set((state) => ({ rnnoiseEnabled: !state.rnnoiseEnabled })),
deafenedUserIds: new Set(),
setUserDeafened: (userId, deafened) => {
set((state) => {
@@ -207,7 +206,7 @@ export const useVoiceStore = create()(persist((set, get) => ({
}),
}), {
name: 'opencord-voice-settings',
version: 2,
version: 3,
migrate: (persistedState, version) => {
if (version === 0) {
persistedState.streamAttenuationEnabled = false;
@@ -216,6 +215,9 @@ export const useVoiceStore = create()(persist((set, get) => ({
persistedState.echoCancellation = true;
persistedState.autoGainControl = false;
}
if (version < 3) {
persistedState.rnnoiseEnabled = false;
}
return persistedState;
},
storage: createJSONStorage(() => localStorage),
@@ -232,6 +234,7 @@ export const useVoiceStore = create()(persist((set, get) => ({
noiseSuppression: state.noiseSuppression,
echoCancellation: state.echoCancellation,
autoGainControl: state.autoGainControl,
rnnoiseEnabled: state.rnnoiseEnabled,
streamAttenuationEnabled: state.streamAttenuationEnabled,
streamAttenuationStrength: state.streamAttenuationStrength,
}),
packages/web/src/stores/voiceStore.ts
+11 -6
View File
@@ -53,7 +53,7 @@ interface VoiceState {
setIsLiveKitConnected: (connected: boolean) => void;
setInputVolume: (volume: number) => void;
setOutputVolume: (volume: number) => void;
setInputDevice: (deviceId: string) => Promise<void>;
setInputDevice: (deviceId: string) => void;
setOutputDevice: (deviceId: string) => void;
toggleMic: () => void;
toggleCamera: () => void;
@@ -64,9 +64,11 @@ interface VoiceState {
noiseSuppression: boolean;
echoCancellation: boolean;
autoGainControl: boolean;
rnnoiseEnabled: boolean;
toggleNoiseSuppression: () => void;
setEchoCancellation: (enabled: boolean) => void;
setAutoGainControl: (enabled: boolean) => void;
toggleRnnoise: () => void;
deafenedUserIds: Set<string>;
setUserDeafened: (userId: string, deafened: boolean) => void;
// WebSocket-based voice user status (visible without joining LiveKit)
@@ -210,10 +212,7 @@ export const useVoiceStore = create<VoiceState>()(
},
setOutputVolume: (volume) => set({ outputVolume: volume }),
setInputDevice: async (deviceId) => {
set({ inputDeviceId: deviceId });
await AudioManager.getInstance().setInputDevice(deviceId);
},
setInputDevice: (deviceId) => set({ inputDeviceId: deviceId }),
setOutputDevice: (deviceId) => set({ outputDeviceId: deviceId }),
@@ -227,9 +226,11 @@ export const useVoiceStore = create<VoiceState>()(
noiseSuppression: true,
echoCancellation: true,
autoGainControl: false,
rnnoiseEnabled: false,
toggleNoiseSuppression: () => set((state) => ({ noiseSuppression: !state.noiseSuppression })),
setEchoCancellation: (enabled) => set({ echoCancellation: enabled }),
setAutoGainControl: (enabled) => set({ autoGainControl: enabled }),
toggleRnnoise: () => set((state) => ({ rnnoiseEnabled: !state.rnnoiseEnabled })),
deafenedUserIds: new Set(),
setUserDeafened: (userId, deafened) => {
set((state) => {
@@ -304,7 +305,7 @@ export const useVoiceStore = create<VoiceState>()(
}),
{
name: 'opencord-voice-settings',
version: 2,
version: 3,
migrate: (persistedState: any, version: number) => {
if (version === 0) {
persistedState.streamAttenuationEnabled = false;
@@ -313,6 +314,9 @@ export const useVoiceStore = create<VoiceState>()(
persistedState.echoCancellation = true;
persistedState.autoGainControl = false;
}
if (version < 3) {
persistedState.rnnoiseEnabled = false;
}
return persistedState;
},
storage: createJSONStorage(() => localStorage),
@@ -329,6 +333,7 @@ export const useVoiceStore = create<VoiceState>()(
noiseSuppression: state.noiseSuppression,
echoCancellation: state.echoCancellation,
autoGainControl: state.autoGainControl,
rnnoiseEnabled: state.rnnoiseEnabled,
streamAttenuationEnabled: state.streamAttenuationEnabled,
streamAttenuationStrength: state.streamAttenuationStrength,
}),
packages/web/src/vite-env.d.ts
Vendored
+1
View File
@@ -0,0 +1 @@
/// <reference types="vite/client" />
pnpm-lock.yaml
Generated
+8
View File
@@ -105,6 +105,9 @@ importers:
'@opencord/shared':
specifier: workspace:*
version: link:../shared
'@sapphi-red/web-noise-suppressor':
specifier: ^0.3.5
version: 0.3.5
livekit-client:
specifier: ^2.9.0
version: 2.17.1(@types/dom-mediacapture-record@1.0.22)
@@ -1233,6 +1236,9 @@ packages:
cpu: [x64]
os: [win32]
'@sapphi-red/web-noise-suppressor@0.3.5':
resolution: {integrity: sha512-jh3+V9yM+zxLriQexoGm0GatoPaJWjs6ypFIbFYwQp+AoUb55eUXrjKtKQyuC5zShzzeAQUl0M5JzqB7SSrsRA==}
'@sindresorhus/is@4.6.0':
resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==}
engines: {node: '>=10'}
@@ -4870,6 +4876,8 @@ snapshots:
'@rollup/rollup-win32-x64-msvc@4.57.1':
optional: true
'@sapphi-red/web-noise-suppressor@0.3.5': {}
'@sindresorhus/is@4.6.0': {}
'@standard-schema/spec@1.1.0': {}