diff --git a/docs/roadmap-resenha.md b/docs/roadmap-resenha.md index 06e50985..791b5096 100644 --- a/docs/roadmap-resenha.md +++ b/docs/roadmap-resenha.md @@ -84,7 +84,7 @@ modais de convite · configurações restantes · telas de erro | Problema | Causa provável | Correção | |---|---|---| | **Enviar som ao soundboard não funciona no app desktop** (funciona no navegador) | `SoundboardPopover` pede o nome do som com `window.prompt`, que o **Electron não implementa** — não abre nada e devolve vazio, então o fluxo aborta em silêncio, sem erro. `window.prompt` aparece em exatamente um lugar no projeto: esse. O resto do código já o evitava | Trocar por um campo de texto dentro do próprio popover (ou um modal reutilizável). Some o prompt e passa a funcionar igual nos dois. **Vale criar o modal de entrada genérico**, já que não existe nenhum e outras features vão precisar | -| **Eco absurdo ao compartilhar tela com som** (app desktop) | O `loopback` do Electron captura a mistura de saída do sistema **inteiro**, que inclui o próprio Backspace tocando a voz dos outros. Essa voz volta para eles dentro da transmissão, com atraso. Não é eco acústico — é digital, então **fone não resolve**. As constraints do web (`restrictOwnAudio`) nunca chegam: o handler ignora o `_request` e monta o stream a partir do enum. É também por isso que `shareAudio` já vinha desligado por padrão no app | Sem solução limpa no Electron: nem `loopback` nem `loopbackWithMute` excluem o áudio do próprio app. Opções reais na seção abaixo | +| **Eco absurdo ao compartilhar tela com som** (app desktop) | O `loopback` do Electron captura a mistura de saída do sistema **inteiro**, que inclui o próprio Backspace tocando a voz dos outros. Essa voz volta para eles dentro da transmissão, com atraso. Não é eco acústico — é digital, então **fone não resolve**. As constraints do web (`restrictOwnAudio`) nunca chegam: o handler ignora o `_request` e monta o stream a partir do enum. É também por isso que `shareAudio` já vinha desligado por padrão no app | **Implementado (2026-09-01), não testado.** Trocado por `electron-native-screenshare`: captura com isolamento por processo — só a janela compartilhada (include) ou tudo menos este app (exclude). O PCM vai do processo principal ao renderer por IPC e vira faixa publicada no LiveKit. **Exige gerar instalador novo e reinstalar**, e só pode ser confirmado numa máquina Windows | | **Bloco do Spotify dessincronizado** (mostra a faixa errada por um tempo) | Soma de duas esperas: a consulta roda a cada **20s** (`useSpotifyActivity`) e o envio pelo WebSocket ainda passa por um **debounce de 5s** no `activityStore`. Na pior hipótese os outros veem a música anterior por ~25s | Consultar de novo perto do fim da faixa (a duração é conhecida) em vez de só por intervalo fixo, e encurtar o debounce para esta fonte | | **Bloco do Spotify some sozinho** | Três caminhos apagam a atividade inteira: faixa **pausada** (`is_playing: false` devolve `null`), o **vão entre faixas** (o Spotify responde `204`) e qualquer falha transitória. Some e volta = a piscada que você viu | Manter a última faixa conhecida por alguns segundos antes de apagar, e enviar um estado *pausado* explícito em vez de sumir com o bloco | | **Barra de progresso errada / andando pausada** | Duas causas independentes: (1) o progresso é derivado de carimbos calculados com o relógio do **servidor** e desenhado contra o relógio de **quem olha** — se os relógios divergem, a barra fica deslocada; (2) a barra continua avançando localmente depois que a pessoa pausa, até a próxima consulta | Enviar o horário do servidor junto no payload para o cliente corrigir a diferença, e congelar a barra quando o estado for pausado | diff --git a/packages/desktop/package.json b/packages/desktop/package.json index 70c152fa..c4f77e30 100644 --- a/packages/desktop/package.json +++ b/packages/desktop/package.json @@ -21,6 +21,7 @@ "postinstall": "electron-rebuild -f -w uiohook-napi || node -e \"console.warn('[desktop] uiohook-napi native rebuild skipped - needs build tools (make, g++, python3). Only required to RUN the desktop app; the server, web client, and Docker image are unaffected.')\"" }, "dependencies": { + "electron-native-screenshare": "^1.2.0", "electron-updater": "^6.3.0", "uiohook-napi": "^1.5.5" }, diff --git a/packages/desktop/src/main.ts b/packages/desktop/src/main.ts index 9029069d..0a0dbbb5 100644 --- a/packages/desktop/src/main.ts +++ b/packages/desktop/src/main.ts @@ -265,6 +265,102 @@ function applyLoginItemSettings(openAtLogin: boolean, startMinimized: boolean): } } +// ─── Native system-audio capture ──────────────────────────────────────────── +// +// Electron's own `audio: 'loopback'` captures the whole output mix, which +// includes this app playing everyone else's voices — so those voices went back +// out inside the screen share and every listener heard themselves. Not acoustic +// echo: it is a digital copy of the output, so headphones never helped. +// +// This module captures with process-level isolation instead: only the shared +// window (include mode), or everything except this app (exclude mode). +interface NativeAudioMeta { + sampleRate: number; + channels: number; + bitsPerSample: number; + isFloat: boolean; +} + +interface NativeScreenShareAudio { + startCapture(processId?: number, isIncludeMode?: boolean, onData?: (data: Buffer, meta: NativeAudioMeta) => void): boolean; + stopCapture(): boolean; + getPidFromWindowHandle(windowHandle: number): number; + isAvailable(): boolean; + getLoadError(): string | null; +} + +let nativeAudio: NativeScreenShareAudio | null = null; +try { + // Required lazily and defensively: a native module that fails to load must + // degrade to sharing without audio, never stop the app from starting. + nativeAudio = require('electron-native-screenshare') as NativeScreenShareAudio; + if (!nativeAudio.isAvailable()) { + console.warn('[Main:ScreenShare] Native audio unavailable:', nativeAudio.getLoadError()); + nativeAudio = null; + } +} catch (err) { + console.warn('[Main:ScreenShare] Native audio module missing:', err); + nativeAudio = null; +} + +let nativeAudioActive = false; + +/** + * Windows and Linux hand desktopCapturer ids of the form `window::`. + * Recovering the handle lets us capture only that window's audio, which is + * better than excluding ourselves: a game's sound goes out, the rest of the + * desktop does not. + */ +function windowHandleFromSourceId(sourceId: string): number | null { + const match = /^window:(\d+)/.exec(sourceId); + if (!match) return null; + const handle = Number(match[1]); + return Number.isFinite(handle) && handle > 0 ? handle : null; +} + +function startNativeAudioCapture(sourceId: string): boolean { + if (!nativeAudio || nativeAudioActive) return false; + + let targetPid = process.pid; + let includeMode = false; + + const handle = windowHandleFromSourceId(sourceId); + if (handle !== null) { + const pid = nativeAudio.getPidFromWindowHandle(handle); + // pid 0 means the handle did not resolve; fall back to excluding ourselves + // rather than capturing nothing. + if (pid > 0) { + targetPid = pid; + includeMode = true; + } + } + + try { + const started = nativeAudio.startCapture(targetPid, includeMode, (data, meta) => { + if (!mainWindow || mainWindow.isDestroyed()) return; + mainWindow.webContents.send('native-audio-data', data, meta); + }); + nativeAudioActive = started; + console.log('[Main:ScreenShare] Native audio', started ? 'started' : 'failed', + includeMode ? `(only pid ${targetPid})` : '(excluding self)'); + return started; + } catch (err) { + console.error('[Main:ScreenShare] Native audio start failed:', err); + nativeAudioActive = false; + return false; + } +} + +function stopNativeAudioCapture(): void { + if (!nativeAudio || !nativeAudioActive) return; + try { + nativeAudio.stopCapture(); + } catch (err) { + console.warn('[Main:ScreenShare] Native audio stop failed:', err); + } + nativeAudioActive = false; +} + // ─── Tray Icon ────────────────────────────────────────────────────────────── function generateFallbackTrayIcon(): Electron.NativeImage { @@ -594,6 +690,10 @@ function registerIpcHandlers(): void { ipcMain.handle('get-app-version', () => app.getVersion()); // Screen share picker coordination (used by setDisplayMediaRequestHandler) + ipcMain.on('native-audio-stop', () => { + stopNativeAudioCapture(); + }); + ipcMain.on('screen-share-selected', (_event, _sourceId: string | null, _shareAudio?: boolean) => { // Handled via ipcMain.once in the display media handler — this is just // a safety net to prevent unhandled-message warnings @@ -953,7 +1053,19 @@ if (!gotTheLock) { // `PulseaudioLoopbackForScreenShare` feature flag we enable above. // Fails on PipeWire-only systems without pulse compat — the // renderer catches that and toasts the user. - callback({ video: selected, ...(shareAudio ? { audio: 'loopback' } : {}) }); + // Audio no longer rides on the Electron stream: `loopback` would put + // this app's own output (everyone else's voices) back into the share. + // The native module captures it separately, isolated by process, and + // the renderer turns it into the track LiveKit publishes. + if (shareAudio) { + const started = startNativeAudioCapture(sourceId); + if (!started) { + // Tell the renderer so it can say the share is going out silently, + // instead of the user assuming sound is included. + mainWindow?.webContents.send('native-audio-unavailable'); + } + } + callback({ video: selected }); } catch (err) { console.error('[Main:ScreenShare] Handler error:', err); // @ts-ignore — deny the request without crashing diff --git a/packages/desktop/src/preload.ts b/packages/desktop/src/preload.ts index 35730519..92a005cf 100644 --- a/packages/desktop/src/preload.ts +++ b/packages/desktop/src/preload.ts @@ -65,6 +65,24 @@ contextBridge.exposeInMainWorld('backspace', { onScreenShareSources: (callback: (sources: unknown[]) => void) => { ipcRenderer.on('screen-share-sources', (_event, sources) => callback(sources)); }, + // Raw PCM from the native capture. Arrives ~50x/second; the renderer turns it + // into a MediaStreamTrack for LiveKit. + onNativeAudioData: (callback: (data: ArrayBuffer, meta: { sampleRate: number; channels: number; bitsPerSample: number; isFloat: boolean }) => void) => { + const listener = (_event: unknown, data: Uint8Array, meta: { sampleRate: number; channels: number; bitsPerSample: number; isFloat: boolean }) => { + // Copied out of the transferred buffer: reusing it across IPC messages + // would let a later chunk overwrite one still being read. + callback(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer, meta); + }; + ipcRenderer.on('native-audio-data', listener); + return () => ipcRenderer.off('native-audio-data', listener); + }, + onNativeAudioUnavailable: (callback: () => void) => { + const listener = () => callback(); + ipcRenderer.on('native-audio-unavailable', listener); + return () => ipcRenderer.off('native-audio-unavailable', listener); + }, + stopNativeAudio: () => ipcRenderer.send('native-audio-stop'), + selectScreenSource: (sourceId: string | null, shareAudio?: boolean) => { ipcRenderer.send('screen-share-selected', sourceId, shareAudio ?? true); }, diff --git a/packages/web/src/platform/electron.d.ts b/packages/web/src/platform/electron.d.ts index e6437147..da84847d 100644 --- a/packages/web/src/platform/electron.d.ts +++ b/packages/web/src/platform/electron.d.ts @@ -61,6 +61,11 @@ interface BackspaceElectronAPI { // Screen share picker coordination onScreenShareSources: (callback: (sources: ElectronScreenSource[]) => void) => void; selectScreenSource: (sourceId: string | null, shareAudio?: boolean) => void; + onNativeAudioData?: ( + callback: (data: ArrayBuffer, meta: { sampleRate: number; channels: number; bitsPerSample: number; isFloat: boolean }) => void, + ) => () => void; + onNativeAudioUnavailable?: (callback: () => void) => () => void; + stopNativeAudio?: () => void; // Instance URL management getInstanceUrl: () => Promise; diff --git a/packages/web/src/utils/nativeScreenAudio.ts b/packages/web/src/utils/nativeScreenAudio.ts new file mode 100644 index 00000000..e5032fa5 --- /dev/null +++ b/packages/web/src/utils/nativeScreenAudio.ts @@ -0,0 +1,126 @@ +import { Room, Track, LocalAudioTrack } from 'livekit-client'; +import { isElectron } from '../platform/platform'; + +/** + * Turns the desktop app's native system-audio capture into a track LiveKit can + * publish. + * + * The native module hands the main process raw PCM, which arrives here in + * chunks over IPC. Web Audio has no "push samples" input, so each chunk is + * scheduled as a buffer source on a running cursor — the standard way to play a + * live stream without gaps. + * + * Exists because Electron's `audio: 'loopback'` captures the whole output mix, + * this app's own playback included, so every listener heard themselves echo. + */ +class NativeScreenAudio { + private ctx: AudioContext | null = null; + private destination: MediaStreamAudioDestinationNode | null = null; + private track: LocalAudioTrack | null = null; + private unsubscribe: (() => void) | null = null; + private nextStartTime = 0; + + /** + * Held ahead of the cursor so a late chunk does not land in the past and get + * dropped. Small enough not to be noticeable against the video. + */ + private static readonly BUFFER_AHEAD_S = 0.08; + /** Beyond this the stream has stalled; resync rather than drift forever. */ + private static readonly MAX_DRIFT_S = 0.5; + + isActive(): boolean { + return this.track !== null; + } + + async start(room: Room): Promise { + if (!isElectron() || !window.backspace?.onNativeAudioData) return false; + if (this.track) return true; + + this.ctx = new AudioContext(); + this.destination = this.ctx.createMediaStreamDestination(); + this.nextStartTime = 0; + + this.unsubscribe = window.backspace.onNativeAudioData((data, meta) => { + this.enqueue(data, meta); + }); + + const mediaTrack = this.destination.stream.getAudioTracks()[0]; + if (!mediaTrack) { + await this.stop(); + return false; + } + + this.track = new LocalAudioTrack(mediaTrack); + try { + await room.localParticipant.publishTrack(this.track, { + source: Track.Source.ScreenShareAudio, + }); + return true; + } catch (err) { + console.error('[NativeScreenAudio] publish failed', err); + await this.stop(); + return false; + } + } + + private enqueue(data: ArrayBuffer, meta: { sampleRate: number; channels: number; isFloat: boolean; bitsPerSample: number }): void { + const ctx = this.ctx; + const destination = this.destination; + if (!ctx || !destination) return; + + // Only float32 is handled: it is what every supported platform reports. + // Anything else is dropped rather than played as noise. + if (!meta.isFloat || meta.bitsPerSample !== 32) return; + + const samples = new Float32Array(data); + const channels = Math.max(1, meta.channels); + const frames = Math.floor(samples.length / channels); + if (frames === 0) return; + + const buffer = ctx.createBuffer(channels, frames, meta.sampleRate); + for (let channel = 0; channel < channels; channel++) { + const channelData = buffer.getChannelData(channel); + // Interleaved in, planar out. The bounds are computed from the buffer's + // own length, but the index signature is still optional under + // noUncheckedIndexedAccess — a truncated final chunk reads as silence + // rather than NaN, which would click. + for (let frame = 0; frame < frames; frame++) { + channelData[frame] = samples[frame * channels + channel] ?? 0; + } + } + + const source = ctx.createBufferSource(); + source.buffer = buffer; + source.connect(destination); + + const now = ctx.currentTime; + if (this.nextStartTime < now || this.nextStartTime > now + NativeScreenAudio.MAX_DRIFT_S) { + this.nextStartTime = now + NativeScreenAudio.BUFFER_AHEAD_S; + } + source.start(this.nextStartTime); + this.nextStartTime += buffer.duration; + } + + async stop(): Promise { + this.unsubscribe?.(); + this.unsubscribe = null; + window.backspace?.stopNativeAudio?.(); + + if (this.track) { + try { + this.track.stop(); + } catch { /* already gone */ } + this.track = null; + } + this.destination = null; + if (this.ctx) { + try { + await this.ctx.close(); + } catch { /* already closed */ } + this.ctx = null; + } + this.nextStartTime = 0; + } +} + +export const nativeScreenAudio = new NativeScreenAudio(); diff --git a/packages/web/src/utils/screenShare.ts b/packages/web/src/utils/screenShare.ts index 0915637a..d10f054c 100644 --- a/packages/web/src/utils/screenShare.ts +++ b/packages/web/src/utils/screenShare.ts @@ -1,5 +1,7 @@ import { Room, Track, BackupCodecPolicy } from 'livekit-client'; import { useVoiceStore } from '../stores/voiceStore'; +import { nativeScreenAudio } from './nativeScreenAudio'; +import { isElectron } from '../platform/platform'; import type { ScreenShareConfig } from '../stores/voiceStore'; import { getStreamingLimits } from '../stores/settingsStore'; import { getPublisherPC, getMediaStreamTrack } from './livekitInternals'; @@ -253,7 +255,10 @@ export async function startScreenShare(room: Room): Promise { try { // For native mode: omit resolution constraint to capture at display's full native resolution const captureOptions: any = { - audio: config.shareAudio ? { + // In the desktop app audio never comes through this stream: Electron + // ignores these constraints and its loopback would carry our own output + // back into the share. The native capture below supplies it instead. + audio: config.shareAudio && !isElectron() ? { // Chrome 141+: exclude this tab's own audio from system audio capture // @ts-ignore — restrictOwnAudio is not yet in all TS type definitions restrictOwnAudio: true, @@ -287,6 +292,14 @@ export async function startScreenShare(room: Room): Promise { return false; } + // Desktop only, and only after the video track exists: publishing the audio + // first would briefly show a share with sound and no picture. A failure + // here leaves the share running silently rather than tearing it down. + if (isElectron() && config.shareAudio) { + const ok = await nativeScreenAudio.start(room); + if (!ok) console.warn('[SS] native system audio unavailable — sharing without sound'); + } + // Set content hint from builder (motion for gaming, detail for text) const screenPub = room.localParticipant.getTrackPublications() .find(p => p.source === Track.Source.ScreenShare); @@ -416,6 +429,9 @@ function scheduleEncoderDetection(room: Room): void { // --------------------------------------------------------------------------- export async function stopScreenShare(room: Room): Promise { + // Stopped first: leaving the native capture running would keep reading system + // audio after the share is gone. + await nativeScreenAudio.stop(); try { await room.localParticipant.setScreenShareEnabled(false); } catch (err) {