feat(desktop): capture system audio with process isolation
Electron's audio: 'loopback' captures the whole output mix, this app's own playback included — so everyone else's voices went back out inside the share and each listener heard themselves. Not acoustic echo but a digital copy of the output, which is why headphones never helped, and why shareAudio already defaulted to off in the desktop app. Electron offers no way to exclude our own audio: the docs allow only 'loopback' or 'loopbackWithMute', and the handler discards the renderer's constraints (restrictOwnAudio never arrives). electron-native-screenshare does it at the OS level — WASAPI process loopback on Windows — capturing only the shared window when its pid resolves, and otherwise everything except us. The module hands raw PCM to the main process, so it crosses IPC and is scheduled onto a running cursor in Web Audio to become a MediaStreamTrack, published as ScreenShareAudio. Loading is optional and failure degrades to a silent share rather than blocking the app or the screen share. The browser path is untouched: Chrome honours restrictOwnAudio and has no echo. Verified by typecheck (web and Electron main) and the web suite. The audio path itself cannot be exercised here — no Windows, no Electron, no audio device.
This commit is contained in:
+5
@@ -61,6 +61,11 @@ interface BackspaceElectronAPI {
|
||||
// Screen share picker coordination
|
||||
onScreenShareSources: (callback: (sources: ElectronScreenSource[]) => void) => void;
|
||||
selectScreenSource: (sourceId: string | null, shareAudio?: boolean) => void;
|
||||
onNativeAudioData?: (
|
||||
callback: (data: ArrayBuffer, meta: { sampleRate: number; channels: number; bitsPerSample: number; isFloat: boolean }) => void,
|
||||
) => () => void;
|
||||
onNativeAudioUnavailable?: (callback: () => void) => () => void;
|
||||
stopNativeAudio?: () => void;
|
||||
|
||||
// Instance URL management
|
||||
getInstanceUrl: () => Promise<string | null>;
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import { Room, Track, LocalAudioTrack } from 'livekit-client';
|
||||
import { isElectron } from '../platform/platform';
|
||||
|
||||
/**
|
||||
* Turns the desktop app's native system-audio capture into a track LiveKit can
|
||||
* publish.
|
||||
*
|
||||
* The native module hands the main process raw PCM, which arrives here in
|
||||
* chunks over IPC. Web Audio has no "push samples" input, so each chunk is
|
||||
* scheduled as a buffer source on a running cursor — the standard way to play a
|
||||
* live stream without gaps.
|
||||
*
|
||||
* Exists because Electron's `audio: 'loopback'` captures the whole output mix,
|
||||
* this app's own playback included, so every listener heard themselves echo.
|
||||
*/
|
||||
class NativeScreenAudio {
|
||||
private ctx: AudioContext | null = null;
|
||||
private destination: MediaStreamAudioDestinationNode | null = null;
|
||||
private track: LocalAudioTrack | null = null;
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
private nextStartTime = 0;
|
||||
|
||||
/**
|
||||
* Held ahead of the cursor so a late chunk does not land in the past and get
|
||||
* dropped. Small enough not to be noticeable against the video.
|
||||
*/
|
||||
private static readonly BUFFER_AHEAD_S = 0.08;
|
||||
/** Beyond this the stream has stalled; resync rather than drift forever. */
|
||||
private static readonly MAX_DRIFT_S = 0.5;
|
||||
|
||||
isActive(): boolean {
|
||||
return this.track !== null;
|
||||
}
|
||||
|
||||
async start(room: Room): Promise<boolean> {
|
||||
if (!isElectron() || !window.backspace?.onNativeAudioData) return false;
|
||||
if (this.track) return true;
|
||||
|
||||
this.ctx = new AudioContext();
|
||||
this.destination = this.ctx.createMediaStreamDestination();
|
||||
this.nextStartTime = 0;
|
||||
|
||||
this.unsubscribe = window.backspace.onNativeAudioData((data, meta) => {
|
||||
this.enqueue(data, meta);
|
||||
});
|
||||
|
||||
const mediaTrack = this.destination.stream.getAudioTracks()[0];
|
||||
if (!mediaTrack) {
|
||||
await this.stop();
|
||||
return false;
|
||||
}
|
||||
|
||||
this.track = new LocalAudioTrack(mediaTrack);
|
||||
try {
|
||||
await room.localParticipant.publishTrack(this.track, {
|
||||
source: Track.Source.ScreenShareAudio,
|
||||
});
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error('[NativeScreenAudio] publish failed', err);
|
||||
await this.stop();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private enqueue(data: ArrayBuffer, meta: { sampleRate: number; channels: number; isFloat: boolean; bitsPerSample: number }): void {
|
||||
const ctx = this.ctx;
|
||||
const destination = this.destination;
|
||||
if (!ctx || !destination) return;
|
||||
|
||||
// Only float32 is handled: it is what every supported platform reports.
|
||||
// Anything else is dropped rather than played as noise.
|
||||
if (!meta.isFloat || meta.bitsPerSample !== 32) return;
|
||||
|
||||
const samples = new Float32Array(data);
|
||||
const channels = Math.max(1, meta.channels);
|
||||
const frames = Math.floor(samples.length / channels);
|
||||
if (frames === 0) return;
|
||||
|
||||
const buffer = ctx.createBuffer(channels, frames, meta.sampleRate);
|
||||
for (let channel = 0; channel < channels; channel++) {
|
||||
const channelData = buffer.getChannelData(channel);
|
||||
// Interleaved in, planar out. The bounds are computed from the buffer's
|
||||
// own length, but the index signature is still optional under
|
||||
// noUncheckedIndexedAccess — a truncated final chunk reads as silence
|
||||
// rather than NaN, which would click.
|
||||
for (let frame = 0; frame < frames; frame++) {
|
||||
channelData[frame] = samples[frame * channels + channel] ?? 0;
|
||||
}
|
||||
}
|
||||
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(destination);
|
||||
|
||||
const now = ctx.currentTime;
|
||||
if (this.nextStartTime < now || this.nextStartTime > now + NativeScreenAudio.MAX_DRIFT_S) {
|
||||
this.nextStartTime = now + NativeScreenAudio.BUFFER_AHEAD_S;
|
||||
}
|
||||
source.start(this.nextStartTime);
|
||||
this.nextStartTime += buffer.duration;
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.unsubscribe?.();
|
||||
this.unsubscribe = null;
|
||||
window.backspace?.stopNativeAudio?.();
|
||||
|
||||
if (this.track) {
|
||||
try {
|
||||
this.track.stop();
|
||||
} catch { /* already gone */ }
|
||||
this.track = null;
|
||||
}
|
||||
this.destination = null;
|
||||
if (this.ctx) {
|
||||
try {
|
||||
await this.ctx.close();
|
||||
} catch { /* already closed */ }
|
||||
this.ctx = null;
|
||||
}
|
||||
this.nextStartTime = 0;
|
||||
}
|
||||
}
|
||||
|
||||
export const nativeScreenAudio = new NativeScreenAudio();
|
||||
@@ -1,5 +1,7 @@
|
||||
import { Room, Track, BackupCodecPolicy } from 'livekit-client';
|
||||
import { useVoiceStore } from '../stores/voiceStore';
|
||||
import { nativeScreenAudio } from './nativeScreenAudio';
|
||||
import { isElectron } from '../platform/platform';
|
||||
import type { ScreenShareConfig } from '../stores/voiceStore';
|
||||
import { getStreamingLimits } from '../stores/settingsStore';
|
||||
import { getPublisherPC, getMediaStreamTrack } from './livekitInternals';
|
||||
@@ -253,7 +255,10 @@ export async function startScreenShare(room: Room): Promise<boolean> {
|
||||
try {
|
||||
// For native mode: omit resolution constraint to capture at display's full native resolution
|
||||
const captureOptions: any = {
|
||||
audio: config.shareAudio ? {
|
||||
// In the desktop app audio never comes through this stream: Electron
|
||||
// ignores these constraints and its loopback would carry our own output
|
||||
// back into the share. The native capture below supplies it instead.
|
||||
audio: config.shareAudio && !isElectron() ? {
|
||||
// Chrome 141+: exclude this tab's own audio from system audio capture
|
||||
// @ts-ignore — restrictOwnAudio is not yet in all TS type definitions
|
||||
restrictOwnAudio: true,
|
||||
@@ -287,6 +292,14 @@ export async function startScreenShare(room: Room): Promise<boolean> {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Desktop only, and only after the video track exists: publishing the audio
|
||||
// first would briefly show a share with sound and no picture. A failure
|
||||
// here leaves the share running silently rather than tearing it down.
|
||||
if (isElectron() && config.shareAudio) {
|
||||
const ok = await nativeScreenAudio.start(room);
|
||||
if (!ok) console.warn('[SS] native system audio unavailable — sharing without sound');
|
||||
}
|
||||
|
||||
// Set content hint from builder (motion for gaming, detail for text)
|
||||
const screenPub = room.localParticipant.getTrackPublications()
|
||||
.find(p => p.source === Track.Source.ScreenShare);
|
||||
@@ -416,6 +429,9 @@ function scheduleEncoderDetection(room: Room): void {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function stopScreenShare(room: Room): Promise<void> {
|
||||
// Stopped first: leaving the native capture running would keep reading system
|
||||
// audio after the share is gone.
|
||||
await nativeScreenAudio.stop();
|
||||
try {
|
||||
await room.localParticipant.setScreenShareEnabled(false);
|
||||
} catch (err) {
|
||||
|
||||
Reference in New Issue
Block a user