feat(desktop): capture system audio with process isolation
Electron's audio: 'loopback' captures the whole output mix, this app's own playback included — so everyone else's voices went back out inside the share and each listener heard themselves. Not acoustic echo but a digital copy of the output, which is why headphones never helped, and why shareAudio already defaulted to off in the desktop app. Electron offers no way to exclude our own audio: the docs allow only 'loopback' or 'loopbackWithMute', and the handler discards the renderer's constraints (restrictOwnAudio never arrives). electron-native-screenshare does it at the OS level — WASAPI process loopback on Windows — capturing only the shared window when its pid resolves, and otherwise everything except us. The module hands raw PCM to the main process, so it crosses IPC and is scheduled onto a running cursor in Web Audio to become a MediaStreamTrack, published as ScreenShareAudio. Loading is optional and failure degrades to a silent share rather than blocking the app or the screen share. The browser path is untouched: Chrome honours restrictOwnAudio and has no echo. Verified by typecheck (web and Electron main) and the web suite. The audio path itself cannot be exercised here — no Windows, no Electron, no audio device.
This commit is contained in:
@@ -21,6 +21,7 @@
|
||||
"postinstall": "electron-rebuild -f -w uiohook-napi || node -e \"console.warn('[desktop] uiohook-napi native rebuild skipped - needs build tools (make, g++, python3). Only required to RUN the desktop app; the server, web client, and Docker image are unaffected.')\""
|
||||
},
|
||||
"dependencies": {
|
||||
"electron-native-screenshare": "^1.2.0",
|
||||
"electron-updater": "^6.3.0",
|
||||
"uiohook-napi": "^1.5.5"
|
||||
},
|
||||
|
||||
@@ -265,6 +265,102 @@ function applyLoginItemSettings(openAtLogin: boolean, startMinimized: boolean):
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Native system-audio capture ────────────────────────────────────────────
|
||||
//
|
||||
// Electron's own `audio: 'loopback'` captures the whole output mix, which
|
||||
// includes this app playing everyone else's voices — so those voices went back
|
||||
// out inside the screen share and every listener heard themselves. Not acoustic
|
||||
// echo: it is a digital copy of the output, so headphones never helped.
|
||||
//
|
||||
// This module captures with process-level isolation instead: only the shared
|
||||
// window (include mode), or everything except this app (exclude mode).
|
||||
interface NativeAudioMeta {
|
||||
sampleRate: number;
|
||||
channels: number;
|
||||
bitsPerSample: number;
|
||||
isFloat: boolean;
|
||||
}
|
||||
|
||||
interface NativeScreenShareAudio {
|
||||
startCapture(processId?: number, isIncludeMode?: boolean, onData?: (data: Buffer, meta: NativeAudioMeta) => void): boolean;
|
||||
stopCapture(): boolean;
|
||||
getPidFromWindowHandle(windowHandle: number): number;
|
||||
isAvailable(): boolean;
|
||||
getLoadError(): string | null;
|
||||
}
|
||||
|
||||
let nativeAudio: NativeScreenShareAudio | null = null;
|
||||
try {
|
||||
// Required lazily and defensively: a native module that fails to load must
|
||||
// degrade to sharing without audio, never stop the app from starting.
|
||||
nativeAudio = require('electron-native-screenshare') as NativeScreenShareAudio;
|
||||
if (!nativeAudio.isAvailable()) {
|
||||
console.warn('[Main:ScreenShare] Native audio unavailable:', nativeAudio.getLoadError());
|
||||
nativeAudio = null;
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('[Main:ScreenShare] Native audio module missing:', err);
|
||||
nativeAudio = null;
|
||||
}
|
||||
|
||||
let nativeAudioActive = false;
|
||||
|
||||
/**
|
||||
* Windows and Linux hand desktopCapturer ids of the form `window:<handle>:<n>`.
|
||||
* Recovering the handle lets us capture only that window's audio, which is
|
||||
* better than excluding ourselves: a game's sound goes out, the rest of the
|
||||
* desktop does not.
|
||||
*/
|
||||
function windowHandleFromSourceId(sourceId: string): number | null {
|
||||
const match = /^window:(\d+)/.exec(sourceId);
|
||||
if (!match) return null;
|
||||
const handle = Number(match[1]);
|
||||
return Number.isFinite(handle) && handle > 0 ? handle : null;
|
||||
}
|
||||
|
||||
function startNativeAudioCapture(sourceId: string): boolean {
|
||||
if (!nativeAudio || nativeAudioActive) return false;
|
||||
|
||||
let targetPid = process.pid;
|
||||
let includeMode = false;
|
||||
|
||||
const handle = windowHandleFromSourceId(sourceId);
|
||||
if (handle !== null) {
|
||||
const pid = nativeAudio.getPidFromWindowHandle(handle);
|
||||
// pid 0 means the handle did not resolve; fall back to excluding ourselves
|
||||
// rather than capturing nothing.
|
||||
if (pid > 0) {
|
||||
targetPid = pid;
|
||||
includeMode = true;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const started = nativeAudio.startCapture(targetPid, includeMode, (data, meta) => {
|
||||
if (!mainWindow || mainWindow.isDestroyed()) return;
|
||||
mainWindow.webContents.send('native-audio-data', data, meta);
|
||||
});
|
||||
nativeAudioActive = started;
|
||||
console.log('[Main:ScreenShare] Native audio', started ? 'started' : 'failed',
|
||||
includeMode ? `(only pid ${targetPid})` : '(excluding self)');
|
||||
return started;
|
||||
} catch (err) {
|
||||
console.error('[Main:ScreenShare] Native audio start failed:', err);
|
||||
nativeAudioActive = false;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function stopNativeAudioCapture(): void {
|
||||
if (!nativeAudio || !nativeAudioActive) return;
|
||||
try {
|
||||
nativeAudio.stopCapture();
|
||||
} catch (err) {
|
||||
console.warn('[Main:ScreenShare] Native audio stop failed:', err);
|
||||
}
|
||||
nativeAudioActive = false;
|
||||
}
|
||||
|
||||
// ─── Tray Icon ──────────────────────────────────────────────────────────────
|
||||
|
||||
function generateFallbackTrayIcon(): Electron.NativeImage {
|
||||
@@ -594,6 +690,10 @@ function registerIpcHandlers(): void {
|
||||
ipcMain.handle('get-app-version', () => app.getVersion());
|
||||
|
||||
// Screen share picker coordination (used by setDisplayMediaRequestHandler)
|
||||
ipcMain.on('native-audio-stop', () => {
|
||||
stopNativeAudioCapture();
|
||||
});
|
||||
|
||||
ipcMain.on('screen-share-selected', (_event, _sourceId: string | null, _shareAudio?: boolean) => {
|
||||
// Handled via ipcMain.once in the display media handler — this is just
|
||||
// a safety net to prevent unhandled-message warnings
|
||||
@@ -953,7 +1053,19 @@ if (!gotTheLock) {
|
||||
// `PulseaudioLoopbackForScreenShare` feature flag we enable above.
|
||||
// Fails on PipeWire-only systems without pulse compat — the
|
||||
// renderer catches that and toasts the user.
|
||||
callback({ video: selected, ...(shareAudio ? { audio: 'loopback' } : {}) });
|
||||
// Audio no longer rides on the Electron stream: `loopback` would put
|
||||
// this app's own output (everyone else's voices) back into the share.
|
||||
// The native module captures it separately, isolated by process, and
|
||||
// the renderer turns it into the track LiveKit publishes.
|
||||
if (shareAudio) {
|
||||
const started = startNativeAudioCapture(sourceId);
|
||||
if (!started) {
|
||||
// Tell the renderer so it can say the share is going out silently,
|
||||
// instead of the user assuming sound is included.
|
||||
mainWindow?.webContents.send('native-audio-unavailable');
|
||||
}
|
||||
}
|
||||
callback({ video: selected });
|
||||
} catch (err) {
|
||||
console.error('[Main:ScreenShare] Handler error:', err);
|
||||
// @ts-ignore — deny the request without crashing
|
||||
|
||||
@@ -65,6 +65,24 @@ contextBridge.exposeInMainWorld('backspace', {
|
||||
onScreenShareSources: (callback: (sources: unknown[]) => void) => {
|
||||
ipcRenderer.on('screen-share-sources', (_event, sources) => callback(sources));
|
||||
},
|
||||
// Raw PCM from the native capture. Arrives ~50x/second; the renderer turns it
|
||||
// into a MediaStreamTrack for LiveKit.
|
||||
onNativeAudioData: (callback: (data: ArrayBuffer, meta: { sampleRate: number; channels: number; bitsPerSample: number; isFloat: boolean }) => void) => {
|
||||
const listener = (_event: unknown, data: Uint8Array, meta: { sampleRate: number; channels: number; bitsPerSample: number; isFloat: boolean }) => {
|
||||
// Copied out of the transferred buffer: reusing it across IPC messages
|
||||
// would let a later chunk overwrite one still being read.
|
||||
callback(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer, meta);
|
||||
};
|
||||
ipcRenderer.on('native-audio-data', listener);
|
||||
return () => ipcRenderer.off('native-audio-data', listener);
|
||||
},
|
||||
onNativeAudioUnavailable: (callback: () => void) => {
|
||||
const listener = () => callback();
|
||||
ipcRenderer.on('native-audio-unavailable', listener);
|
||||
return () => ipcRenderer.off('native-audio-unavailable', listener);
|
||||
},
|
||||
stopNativeAudio: () => ipcRenderer.send('native-audio-stop'),
|
||||
|
||||
selectScreenSource: (sourceId: string | null, shareAudio?: boolean) => {
|
||||
ipcRenderer.send('screen-share-selected', sourceId, shareAudio ?? true);
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user