From ccf2001047a60bba55b66ab98bff52c8fff33c29 Mon Sep 17 00:00:00 2001 From: Jannis Braun <151788261+TheZwiss@users.noreply.github.com> Date: Mon, 23 Feb 2026 00:28:29 +0100 Subject: [PATCH] feat: RNNoise WASM AudioWorklet for ML-based noise suppression (Phase 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inject @sapphi-red/web-noise-suppressor into AudioManager's input pipeline as a toggleable AudioWorkletNode. The worklet is loaded lazily on first enable, then kept alive — toggling bypasses by rewiring the graph without destroying the WASM instance. Browser NS is forced off when RNNoise is active to avoid double-processing. InputGain forced to stereo up-mix to prevent mono-left-only output from the worklet. Also wires Phase 2 output device routing (setSinkId) into AppLayout/ChannelSidebar. --- packages/web/package.json | 1 + packages/web/src/audio/AudioManager.js | 120 +++++++++++++-- packages/web/src/audio/AudioManager.ts | 137 ++++++++++++++++-- .../web/src/components/layout/AppLayout.js | 4 + .../web/src/components/layout/AppLayout.tsx | 8 + .../src/components/layout/ChannelSidebar.js | 17 ++- .../src/components/layout/ChannelSidebar.tsx | 20 ++- .../web/src/components/voice/VoiceControls.js | 6 +- .../src/components/voice/VoiceControls.tsx | 34 ++++- packages/web/src/hooks/useLiveKit.js | 4 +- packages/web/src/hooks/useLiveKit.ts | 4 +- packages/web/src/stores/voiceStore.js | 13 +- packages/web/src/stores/voiceStore.ts | 17 ++- packages/web/src/vite-env.d.ts | 1 + pnpm-lock.yaml | 8 + 15 files changed, 345 insertions(+), 49 deletions(-) create mode 100644 packages/web/src/vite-env.d.ts diff --git a/packages/web/package.json b/packages/web/package.json index 011d8897..5ab451de 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -11,6 +11,7 @@ "dependencies": { "@livekit/components-react": "^2.7.4", "@opencord/shared": "workspace:*", + "@sapphi-red/web-noise-suppressor": "^0.3.5", "livekit-client": "^2.9.0", "react": "^18.3.1", "react-dom": "^18.3.1", diff --git a/packages/web/src/audio/AudioManager.js b/packages/web/src/audio/AudioManager.js index 62d6e403..831e86df 100644 --- a/packages/web/src/audio/AudioManager.js +++ b/packages/web/src/audio/AudioManager.js @@ -1,3 +1,8 @@ +import { RnnoiseWorkletNode, loadRnnoise } from '@sapphi-red/web-noise-suppressor'; +import rnnoiseWorkletPath from '@sapphi-red/web-noise-suppressor/rnnoiseWorklet.js?url'; +import rnnoiseWasmPath from '@sapphi-red/web-noise-suppressor/rnnoise.wasm?url'; +import rnnoiseWasmSimdPath from '@sapphi-red/web-noise-suppressor/rnnoise_simd.wasm?url'; + export class AudioManager { static instance = null; ctx = null; @@ -8,6 +13,7 @@ export class AudioManager { analyser = null; masterCompressor = null; currentInputDeviceId = 'default'; + desiredOutputDeviceId = 'default'; currentStream = null; isInitialized = false; listeners = new Set(); @@ -15,7 +21,12 @@ export class AudioManager { voiceEchoCancellation = true; voiceNoiseSuppression = true; voiceAutoGainControl = false; + screenShareActive = false; streamGeneration = 0; + inputSwitchChain = Promise.resolve(null); + rnnoiseNode = null; + rnnoiseEnabled = false; + rnnoiseReady = false; constructor() { } static getInstance() { if (!AudioManager.instance) { @@ -27,8 +38,13 @@ export class AudioManager { if (this.ctx) return; const AudioContextClass = window.AudioContext || window.webkitAudioContext; - this.ctx = new AudioContextClass(); + this.ctx = new AudioContextClass({ sampleRate: 48000 }); this.inputGain = this.ctx.createGain(); + // Force stereo up-mix so mono sources (e.g. RNNoise worklet output) + // are duplicated to both L+R channels instead of left-only. + this.inputGain.channelCount = 2; + this.inputGain.channelCountMode = 'explicit'; + this.inputGain.channelInterpretation = 'speakers'; this.inputDestination = this.ctx.createMediaStreamDestination(); this.analyser = this.ctx.createAnalyser(); this.analyser.fftSize = 256; @@ -55,6 +71,22 @@ export class AudioManager { } }; this.isInitialized = true; + this.applyOutputDevice(); + } + async setOutputDevice(deviceId) { + this.desiredOutputDeviceId = deviceId; + await this.applyOutputDevice(); + } + async applyOutputDevice() { + if (!this.ctx || !('setSinkId' in this.ctx)) return; + try { + const sinkId = this.desiredOutputDeviceId === 'default' ? '' : this.desiredOutputDeviceId; + await this.ctx.setSinkId(sinkId); + console.log(`[AudioManager] Output device set to: ${this.desiredOutputDeviceId}`); + } + catch (err) { + console.error('[AudioManager] Failed to set output device:', err); + } } onResumed(cb) { this.listeners.add(cb); @@ -112,6 +144,11 @@ export class AudioManager { return source; } async setInputDevice(deviceId) { + const job = this.inputSwitchChain.then(() => this._setInputDeviceImpl(deviceId)); + this.inputSwitchChain = job.catch(() => null); + return job; + } + async _setInputDeviceImpl(deviceId) { if (!this.isInitialized) this.initContext(); // Skip if already set and stream is active @@ -122,12 +159,20 @@ export class AudioManager { if (this.currentStream) { this.currentStream.getTracks().forEach(t => t.stop()); } + const effectiveEchoCancellation = this.screenShareActive ? false : this.voiceEchoCancellation; + // When RNNoise is active, force browser NS off — running both degrades quality. + const effectiveNoiseSuppression = this.rnnoiseEnabled ? false : this.voiceNoiseSuppression; const constraints = { audio: { deviceId: deviceId === 'default' ? undefined : { exact: deviceId }, - echoCancellation: this.voiceEchoCancellation, - noiseSuppression: this.voiceNoiseSuppression, + echoCancellation: effectiveEchoCancellation, + noiseSuppression: effectiveNoiseSuppression, autoGainControl: this.voiceAutoGainControl, + googEchoCancellation: effectiveEchoCancellation, + googAutoGainControl: this.voiceAutoGainControl, + googNoiseSuppression: effectiveNoiseSuppression, + googHighpassFilter: false, + googTypingNoiseDetection: false, } }; this.currentStream = await navigator.mediaDevices.getUserMedia(constraints); @@ -138,7 +183,7 @@ export class AudioManager { this.inputSource.disconnect(); } this.inputSource = this.ctx.createMediaStreamSource(this.currentStream); - this.inputSource.connect(this.inputGain); + this.inputSource.connect(this.getInputTarget()); } return this.currentStream; } @@ -147,6 +192,56 @@ export class AudioManager { throw err; } } + getInputTarget() { + return (this.rnnoiseEnabled && this.rnnoiseNode) ? this.rnnoiseNode : this.inputGain; + } + async setRnnoiseEnabled(enabled) { + if (enabled === this.rnnoiseEnabled && this.rnnoiseReady) + return; + if (!this.isInitialized) + this.initContext(); + if (enabled && !this.rnnoiseReady) { + try { + console.log('[AudioManager] Loading RNNoise worklet...'); + await this.ctx.audioWorklet.addModule(rnnoiseWorkletPath); + // loadRnnoise handles SIMD feature detection and returns the right binary + const wasmBinary = await loadRnnoise({ url: rnnoiseWasmPath, simdUrl: rnnoiseWasmSimdPath }); + this.rnnoiseNode = new RnnoiseWorkletNode(this.ctx, { + wasmBinary, + maxChannels: 1, + }); + this.rnnoiseNode.connect(this.inputGain); + this.rnnoiseReady = true; + console.log('[AudioManager] RNNoise worklet loaded and connected'); + } + catch (err) { + console.error('[AudioManager] Failed to load RNNoise worklet:', err); + this.rnnoiseReady = false; + this.rnnoiseEnabled = false; + if (this.inputSource && this.inputGain) { + this.inputSource.disconnect(); + this.inputSource.connect(this.inputGain); + } + return; + } + } + this.rnnoiseEnabled = enabled; + // Rewire the graph + if (this.inputSource) { + this.inputSource.disconnect(); + this.inputSource.connect(this.getInputTarget()); + console.log(`[AudioManager] RNNoise ${enabled ? 'enabled' : 'bypassed'} — inputSource → ${enabled ? 'rnnoiseNode' : 'inputGain'}`); + } + // Force track re-publish so LiveKit picks up the new pipeline + this.streamGeneration++; + if (this.currentStream) { + this.currentStream.getTracks().forEach(t => t.stop()); + this.currentStream = null; + } + } + isRnnoiseEnabled() { + return this.rnnoiseEnabled; + } setInputVolume(volume) { if (!this.isInitialized) this.initContext(); @@ -174,6 +269,15 @@ export class AudioManager { this.currentStream = null; } } + setScreenShareActive(active) { + if (this.screenShareActive === active) return; + this.screenShareActive = active; + console.log(`[AudioManager] Screen share active: ${active} — ${active ? 'forcing AEC off' : 'restoring user AEC preference'}`); + if (this.currentStream) { + this.currentStream.getTracks().forEach(t => t.stop()); + this.currentStream = null; + } + } getStreamGeneration() { return this.streamGeneration; } @@ -208,16 +312,10 @@ export class AudioManager { this.initContext(); return this.ctx; } - /** - * Returns the master output bus (DynamicsCompressorNode). - * All remote audio (voice, stream) should connect their GainNodes - * to this node instead of directly to ctx.destination. The compressor - * prevents clipping when multiple sources sum together. - */ getMasterOutput() { if (!this.ctx) this.initContext(); - return this.ctx.destination; + return this.masterCompressor; } getContext() { return this.ctx; diff --git a/packages/web/src/audio/AudioManager.ts b/packages/web/src/audio/AudioManager.ts index b96fda03..2308d74f 100644 --- a/packages/web/src/audio/AudioManager.ts +++ b/packages/web/src/audio/AudioManager.ts @@ -1,3 +1,8 @@ +import { RnnoiseWorkletNode, loadRnnoise } from '@sapphi-red/web-noise-suppressor'; +import rnnoiseWorkletPath from '@sapphi-red/web-noise-suppressor/rnnoiseWorklet.js?url'; +import rnnoiseWasmPath from '@sapphi-red/web-noise-suppressor/rnnoise.wasm?url'; +import rnnoiseWasmSimdPath from '@sapphi-red/web-noise-suppressor/rnnoise_simd.wasm?url'; + export class AudioManager { private static instance: AudioManager | null = null; private ctx: AudioContext | null = null; @@ -9,6 +14,7 @@ export class AudioManager { private masterCompressor: DynamicsCompressorNode | null = null; private currentInputDeviceId: string = 'default'; + private desiredOutputDeviceId: string = 'default'; private currentStream: MediaStream | null = null; private isInitialized = false; @@ -19,6 +25,10 @@ export class AudioManager { private voiceAutoGainControl = false; private screenShareActive = false; private streamGeneration = 0; + private inputSwitchChain: Promise = Promise.resolve(null); + private rnnoiseNode: AudioWorkletNode | null = null; + private rnnoiseEnabled = false; + private rnnoiseReady = false; private constructor() {} @@ -32,9 +42,14 @@ export class AudioManager { private initContext() { if (this.ctx) return; const AudioContextClass = window.AudioContext || (window as any).webkitAudioContext; - this.ctx = new AudioContextClass(); + this.ctx = new AudioContextClass({ sampleRate: 48000 }); this.inputGain = this.ctx.createGain(); + // Force stereo up-mix so mono sources (e.g. RNNoise worklet output) + // are duplicated to both L+R channels instead of left-only. + this.inputGain.channelCount = 2; + this.inputGain.channelCountMode = 'explicit'; + this.inputGain.channelInterpretation = 'speakers'; this.inputDestination = this.ctx.createMediaStreamDestination(); this.analyser = this.ctx.createAnalyser(); this.analyser.fftSize = 256; @@ -67,6 +82,31 @@ export class AudioManager { }; this.isInitialized = true; + + // Apply pending output device selection (user preference loaded before context creation) + this.applyOutputDevice(); + } + + /** + * Routes all Web Audio output to the specified device via AudioContext.setSinkId(). + * This is the ONLY correct way to switch output devices when using a custom Web Audio + * pipeline — LiveKit's switchActiveDevice('audiooutput') targets