feat(desktop): capture system audio with process isolation

Electron's audio: 'loopback' captures the whole output mix, this app's own
playback included — so everyone else's voices went back out inside the share
and each listener heard themselves. Not acoustic echo but a digital copy of
the output, which is why headphones never helped, and why shareAudio already
defaulted to off in the desktop app.

Electron offers no way to exclude our own audio: the docs allow only
'loopback' or 'loopbackWithMute', and the handler discards the renderer's
constraints (restrictOwnAudio never arrives). electron-native-screenshare does
it at the OS level — WASAPI process loopback on Windows — capturing only the
shared window when its pid resolves, and otherwise everything except us.

The module hands raw PCM to the main process, so it crosses IPC and is
scheduled onto a running cursor in Web Audio to become a MediaStreamTrack,
published as ScreenShareAudio. Loading is optional and failure degrades to a
silent share rather than blocking the app or the screen share.

The browser path is untouched: Chrome honours restrictOwnAudio and has no echo.

Verified by typecheck (web and Electron main) and the web suite. The audio path
itself cannot be exercised here — no Windows, no Electron, no audio device.
This commit is contained in:
2026-09-01 00:56:34 -03:00
parent 310e9b86be
commit d525bbb8c5
7 changed files with 281 additions and 3 deletions
+5
View File
@@ -61,6 +61,11 @@ interface BackspaceElectronAPI {
// Screen share picker coordination
onScreenShareSources: (callback: (sources: ElectronScreenSource[]) => void) => void;
selectScreenSource: (sourceId: string | null, shareAudio?: boolean) => void;
onNativeAudioData?: (
callback: (data: ArrayBuffer, meta: { sampleRate: number; channels: number; bitsPerSample: number; isFloat: boolean }) => void,
) => () => void;
onNativeAudioUnavailable?: (callback: () => void) => () => void;
stopNativeAudio?: () => void;
// Instance URL management
getInstanceUrl: () => Promise<string | null>;
+126
View File
@@ -0,0 +1,126 @@
import { Room, Track, LocalAudioTrack } from 'livekit-client';
import { isElectron } from '../platform/platform';
/**
* Turns the desktop app's native system-audio capture into a track LiveKit can
* publish.
*
* The native module hands the main process raw PCM, which arrives here in
* chunks over IPC. Web Audio has no "push samples" input, so each chunk is
* scheduled as a buffer source on a running cursor — the standard way to play a
* live stream without gaps.
*
* Exists because Electron's `audio: 'loopback'` captures the whole output mix,
* this app's own playback included, so every listener heard themselves echo.
*/
class NativeScreenAudio {
private ctx: AudioContext | null = null;
private destination: MediaStreamAudioDestinationNode | null = null;
private track: LocalAudioTrack | null = null;
private unsubscribe: (() => void) | null = null;
private nextStartTime = 0;
/**
* Held ahead of the cursor so a late chunk does not land in the past and get
* dropped. Small enough not to be noticeable against the video.
*/
private static readonly BUFFER_AHEAD_S = 0.08;
/** Beyond this the stream has stalled; resync rather than drift forever. */
private static readonly MAX_DRIFT_S = 0.5;
isActive(): boolean {
return this.track !== null;
}
async start(room: Room): Promise<boolean> {
if (!isElectron() || !window.backspace?.onNativeAudioData) return false;
if (this.track) return true;
this.ctx = new AudioContext();
this.destination = this.ctx.createMediaStreamDestination();
this.nextStartTime = 0;
this.unsubscribe = window.backspace.onNativeAudioData((data, meta) => {
this.enqueue(data, meta);
});
const mediaTrack = this.destination.stream.getAudioTracks()[0];
if (!mediaTrack) {
await this.stop();
return false;
}
this.track = new LocalAudioTrack(mediaTrack);
try {
await room.localParticipant.publishTrack(this.track, {
source: Track.Source.ScreenShareAudio,
});
return true;
} catch (err) {
console.error('[NativeScreenAudio] publish failed', err);
await this.stop();
return false;
}
}
private enqueue(data: ArrayBuffer, meta: { sampleRate: number; channels: number; isFloat: boolean; bitsPerSample: number }): void {
const ctx = this.ctx;
const destination = this.destination;
if (!ctx || !destination) return;
// Only float32 is handled: it is what every supported platform reports.
// Anything else is dropped rather than played as noise.
if (!meta.isFloat || meta.bitsPerSample !== 32) return;
const samples = new Float32Array(data);
const channels = Math.max(1, meta.channels);
const frames = Math.floor(samples.length / channels);
if (frames === 0) return;
const buffer = ctx.createBuffer(channels, frames, meta.sampleRate);
for (let channel = 0; channel < channels; channel++) {
const channelData = buffer.getChannelData(channel);
// Interleaved in, planar out. The bounds are computed from the buffer's
// own length, but the index signature is still optional under
// noUncheckedIndexedAccess — a truncated final chunk reads as silence
// rather than NaN, which would click.
for (let frame = 0; frame < frames; frame++) {
channelData[frame] = samples[frame * channels + channel] ?? 0;
}
}
const source = ctx.createBufferSource();
source.buffer = buffer;
source.connect(destination);
const now = ctx.currentTime;
if (this.nextStartTime < now || this.nextStartTime > now + NativeScreenAudio.MAX_DRIFT_S) {
this.nextStartTime = now + NativeScreenAudio.BUFFER_AHEAD_S;
}
source.start(this.nextStartTime);
this.nextStartTime += buffer.duration;
}
async stop(): Promise<void> {
this.unsubscribe?.();
this.unsubscribe = null;
window.backspace?.stopNativeAudio?.();
if (this.track) {
try {
this.track.stop();
} catch { /* already gone */ }
this.track = null;
}
this.destination = null;
if (this.ctx) {
try {
await this.ctx.close();
} catch { /* already closed */ }
this.ctx = null;
}
this.nextStartTime = 0;
}
}
export const nativeScreenAudio = new NativeScreenAudio();
+17 -1
View File
@@ -1,5 +1,7 @@
import { Room, Track, BackupCodecPolicy } from 'livekit-client';
import { useVoiceStore } from '../stores/voiceStore';
import { nativeScreenAudio } from './nativeScreenAudio';
import { isElectron } from '../platform/platform';
import type { ScreenShareConfig } from '../stores/voiceStore';
import { getStreamingLimits } from '../stores/settingsStore';
import { getPublisherPC, getMediaStreamTrack } from './livekitInternals';
@@ -253,7 +255,10 @@ export async function startScreenShare(room: Room): Promise<boolean> {
try {
// For native mode: omit resolution constraint to capture at display's full native resolution
const captureOptions: any = {
audio: config.shareAudio ? {
// In the desktop app audio never comes through this stream: Electron
// ignores these constraints and its loopback would carry our own output
// back into the share. The native capture below supplies it instead.
audio: config.shareAudio && !isElectron() ? {
// Chrome 141+: exclude this tab's own audio from system audio capture
// @ts-ignore — restrictOwnAudio is not yet in all TS type definitions
restrictOwnAudio: true,
@@ -287,6 +292,14 @@ export async function startScreenShare(room: Room): Promise<boolean> {
return false;
}
// Desktop only, and only after the video track exists: publishing the audio
// first would briefly show a share with sound and no picture. A failure
// here leaves the share running silently rather than tearing it down.
if (isElectron() && config.shareAudio) {
const ok = await nativeScreenAudio.start(room);
if (!ok) console.warn('[SS] native system audio unavailable — sharing without sound');
}
// Set content hint from builder (motion for gaming, detail for text)
const screenPub = room.localParticipant.getTrackPublications()
.find(p => p.source === Track.Source.ScreenShare);
@@ -416,6 +429,9 @@ function scheduleEncoderDetection(room: Room): void {
// ---------------------------------------------------------------------------
export async function stopScreenShare(room: Room): Promise<void> {
// Stopped first: leaving the native capture running would keep reading system
// audio after the share is gone.
await nativeScreenAudio.stop();
try {
await room.localParticipant.setScreenShareEnabled(false);
} catch (err) {